improve ca data for yurthub component

pkg/util/certmanager/pki.go

@@ -185,3 +185,10 @@ func GenCertPoolUseCA(caFile string) (*x509.CertPool, error) {
 	certPool.AppendCertsFromPEM(caData)
 	return certPool, nil
 }
+
+// GenCertPoolUseCAData generates a x509 CertPool based on the given CA data
+func GenCertPoolUseCAData(caData []byte) (*x509.CertPool, error) {
+	certPool := x509.NewCertPool()
+	certPool.AppendCertsFromPEM(caData)
+	return certPool, nil
+}

pkg/yurthub/certificate/interfaces.go

@@ -45,6 +45,7 @@ type YurtClientCertificateManager interface {
 	Stop()
 	UpdateBootstrapConf(joinToken string) error
 	GetHubConfFile() string
+	GetCAData() []byte
 	GetCaFile() string
 	GetAPIServerClientCert() *tls.Certificate
 }

pkg/yurthub/certificate/kubeletcertificate/kubelet_certificate.go

@@ -21,6 +21,7 @@ import (
 	"crypto/x509"
 	"errors"
 	"fmt"
+	"os"
 	"time"
 
 	"k8s.io/klog/v2"
@@ -40,6 +41,7 @@ type kubeletCertManager struct {
 	kubeletCAFile  string
 	kubeletPemFile string
 	cert           *tls.Certificate
+	caData         []byte
 }
 
 func NewKubeletCertManager(kubeConfFile, kubeletCAFile, kubeletPemFile string) (certificate.YurtClientCertificateManager, error) {
@@ -50,6 +52,10 @@ func NewKubeletCertManager(kubeConfFile, kubeletCAFile, kubeletPemFile string) (
 	if exist, _ := util.FileExists(kubeletCAFile); !exist {
 		return nil, KubeletCANotExistErr
 	}
+	caData, err := os.ReadFile(kubeletCAFile)
+	if err != nil {
+		return nil, err
+	}
 
 	if exist, _ := util.FileExists(kubeletPemFile); !exist {
 		return nil, KubeletPemNotExistErr
@@ -65,6 +71,7 @@ func NewKubeletCertManager(kubeConfFile, kubeletCAFile, kubeletPemFile string) (
 		kubeletCAFile:  kubeletCAFile,
 		kubeletPemFile: kubeletPemFile,
 		cert:           cert,
+		caData:         caData,
 	}, nil
 }
 
@@ -84,6 +91,10 @@ func (kcm *kubeletCertManager) GetHubConfFile() string {
 	return kcm.kubeConfFile
 }
 
+func (kcm *kubeletCertManager) GetCAData() []byte {
+	return kcm.caData
+}
+
 func (kcm *kubeletCertManager) GetCaFile() string {
 	return kcm.kubeletCAFile
 }

pkg/yurthub/certificate/manager/manager.go

@@ -32,7 +32,6 @@ import (
 	"github.com/openyurtio/openyurt/pkg/yurthub/certificate/kubeletcertificate"
 	hubServerCert "github.com/openyurtio/openyurt/pkg/yurthub/certificate/server"
 	"github.com/openyurtio/openyurt/pkg/yurthub/certificate/token"
-	"github.com/openyurtio/openyurt/pkg/yurthub/util"
 )
 
 const (
@@ -123,12 +122,8 @@ func (hcm *yurtHubCertManager) Ready() bool {
 		errs = append(errs, apiServerClientCertNotReadyError)
 	}
 
-	if exist, err := util.FileExists(hcm.YurtClientCertificateManager.GetCaFile()); !exist {
-		if err == nil {
-			errs = append(errs, caCertIsNotReadyError)
-		} else {
-			errs = append(errs, err)
-		}
+	if len(hcm.YurtClientCertificateManager.GetCAData()) == 0 {
+		errs = append(errs, caCertIsNotReadyError)
 	}
 
 	if hcm.GetHubServerCert() == nil {

pkg/yurthub/certificate/manager/manager_test.go

@@ -24,16 +24,11 @@ import (
 	"testing"
 	"time"
 
-	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/util/wait"
-	"k8s.io/client-go/tools/clientcmd"
-	certutil "k8s.io/client-go/util/cert"
 
 	"github.com/openyurtio/openyurt/cmd/yurthub/app/options"
 	"github.com/openyurtio/openyurt/pkg/projectinfo"
-	kubeconfigutil "github.com/openyurtio/openyurt/pkg/util/kubeconfig"
 	"github.com/openyurtio/openyurt/pkg/yurthub/certificate/testdata"
-	"github.com/openyurtio/openyurt/pkg/yurthub/util"
 )
 
 func TestGetHubServerCertFile(t *testing.T) {
@@ -108,33 +103,6 @@ func TestReady(t *testing.T) {
 		if mgr.Ready() {
 			return true, nil
 		}
-
-		if exist, err := util.FileExists(mgr.GetCaFile()); !exist {
-			if err != nil {
-				t.Logf("could not get ca file(%s), %v", mgr.GetCaFile(), err)
-				return false, err
-			}
-
-			if exist, err := util.FileExists(mgr.GetHubConfFile()); err != nil {
-				t.Logf("could not get hub conf file(%s), %v", mgr.GetHubConfFile(), err)
-				return false, nil
-			} else if exist {
-				t.Logf("%s file already exists, so use it to create ca file", mgr.GetHubConfFile())
-				hubKubeConfig, err := clientcmd.LoadFromFile(mgr.GetHubConfFile())
-				if err != nil {
-					return false, err
-				}
-
-				cluster := kubeconfigutil.GetClusterFromKubeConfig(hubKubeConfig)
-				if cluster != nil {
-					if err := certutil.WriteCert(mgr.GetCaFile(), cluster.CertificateAuthorityData); err != nil {
-						return false, errors.Wrap(err, "couldn't save the CA certificate to disk")
-					}
-				} else {
-					return false, errors.Errorf("couldn't prepare ca.crt(%s) file", mgr.GetCaFile())
-				}
-			}
-		}
 		return false, nil
 	})
 
@@ -161,8 +129,14 @@ func TestReady(t *testing.T) {
 		return
 	}
 	newMgr.Start()
-	if !newMgr.Ready() {
-		t.Errorf("certificates can not be reused")
+	err = wait.PollImmediate(2*time.Second, 1*time.Minute, func() (done bool, err error) {
+		if mgr.Ready() {
+			return true, nil
+		}
+		return false, nil
+	})
+	if err != nil {
+		t.Errorf("certificates are not reused, %v", err)
 	}
 	newMgr.Stop()
 

pkg/yurthub/certificate/token/token.go

@@ -80,6 +80,7 @@ type yurtHubClientCertManager struct {
 	joinToken                  string
 	bootstrapFile              string
 	dialer                     *util.Dialer
+	caData                     []byte
 }
 
 // NewYurtHubClientCertManager new a YurtCertificateManager instance
@@ -203,11 +204,17 @@ func (ycm *yurtHubClientCertManager) prepareConfigAndCaFile() error {
 				if err := certutil.WriteCert(ycm.GetCaFile(), cluster.CertificateAuthorityData); err != nil {
 					return errors.Wrap(err, "couldn't save the CA certificate to disk")
 				}
+				ycm.caData = cluster.CertificateAuthorityData
 			} else {
 				return errors.Errorf("couldn't prepare ca.crt(%s) file", ycm.GetCaFile())
 			}
 		} else {
 			klog.V(2).Infof("%s file already exists, so reuse it", ycm.GetCaFile())
+			caData, err := os.ReadFile(ycm.GetCaFile())
+			if err != nil {
+				return err
+			}
+			ycm.caData = caData
 		}
 		return nil
 	}
@@ -250,11 +257,17 @@ func (ycm *yurtHubClientCertManager) prepareConfigAndCaFile() error {
 			if err := certutil.WriteCert(ycm.GetCaFile(), cluster.CertificateAuthorityData); err != nil {
 				return errors.Wrap(err, "couldn't save the CA certificate to disk")
 			}
+			ycm.caData = cluster.CertificateAuthorityData
 		} else {
 			return errors.Errorf("couldn't prepare ca.crt(%s) file", ycm.GetCaFile())
 		}
 	} else {
 		klog.V(2).Infof("%s file already exists, so reuse it", ycm.GetCaFile())
+		caData, err := os.ReadFile(ycm.GetCaFile())
+		if err != nil {
+			return err
+		}
+		ycm.caData = caData
 	}
 
 	return nil
@@ -284,6 +297,10 @@ func (ycm *yurtHubClientCertManager) getBootstrapConfFile() string {
 	return filepath.Join(ycm.hubRunDir, bootstrapConfigFileName)
 }
 
+func (ycm *yurtHubClientCertManager) GetCAData() []byte {
+	return ycm.caData
+}
+
 // GetCaFile returns the path of ca file
 func (ycm *yurtHubClientCertManager) GetCaFile() string {
 	return filepath.Join(ycm.getPkiDir(), hubCaFileName)

pkg/yurthub/transport/transport.go

@@ -34,8 +34,8 @@ type CertGetter interface {
 	// GetAPIServerClientCert returns the currently selected certificate, as well as
 	// the associated certificate and key data in PEM format.
 	GetAPIServerClientCert() *tls.Certificate
-	// Return CA file path.
-	GetCaFile() string
+	// GetCAData returns CA file data.
+	GetCAData() []byte
 }
 
 // Interface is an transport interface for managing clients that used to connecting kube-apiserver
@@ -60,13 +60,12 @@ type transportManager struct {
 
 // NewTransportManager create a transport interface object.
 func NewTransportManager(certGetter CertGetter, stopCh <-chan struct{}) (Interface, error) {
-	caFile := certGetter.GetCaFile()
-	if len(caFile) == 0 {
-		return nil, fmt.Errorf("ca cert file was not prepared when new transport")
+	caData := certGetter.GetCAData()
+	if len(caData) == 0 {
+		return nil, fmt.Errorf("ca cert data was not prepared when new transport")
 	}
-	klog.V(2).Infof("use %s ca cert file to access remote server", caFile)
 
-	cfg, err := tlsConfig(certGetter.GetAPIServerClientCert, caFile)
+	cfg, err := tlsConfig(certGetter.GetAPIServerClientCert, caData)
 	if err != nil {
 		klog.Errorf("could not get tls config when new transport, %v", err)
 		return nil, err
@@ -81,7 +80,7 @@ func NewTransportManager(certGetter CertGetter, stopCh <-chan struct{}) (Interfa
 		DialContext:         d.DialContext,
 	})
 
-	bearerTLSCfg, err := tlsConfig(nil, caFile)
+	bearerTLSCfg, err := tlsConfig(nil, caData)
 	if err != nil {
 		klog.Errorf("could not get tls config when new bearer transport, %v", err)
 		return nil, err
@@ -151,9 +150,9 @@ func (tm *transportManager) start() {
 	}, 10*time.Second, tm.stopCh)
 }
 
-func tlsConfig(current func() *tls.Certificate, caFile string) (*tls.Config, error) {
+func tlsConfig(current func() *tls.Certificate, caData []byte) (*tls.Config, error) {
 	// generate the TLS configuration based on the latest certificate
-	rootCert, err := certmanager.GenCertPoolUseCA(caFile)
+	rootCert, err := certmanager.GenCertPoolUseCAData(caData)
 	if err != nil {
 		klog.Errorf("could not generate a x509 CertPool based on the given CA file, %v", err)
 		return nil, err

pkg/yurthub/yurtcoordinator/certmanager/certmanager.go

@@ -98,6 +98,7 @@ type CertManager struct {
 	coordinatorCert    *tls.Certificate
 	nodeLeaseProxyCert *tls.Certificate
 	store              fs.FileSystemOperator
+	caData             []byte
 
 	// Used for unit test.
 	secret *corev1.Secret
@@ -115,6 +116,10 @@ func (c *CertManager) GetNodeLeaseProxyClientCert() *tls.Certificate {
 	return c.nodeLeaseProxyCert
 }
 
+func (c *CertManager) GetCAData() []byte {
+	return c.caData
+}
+
 func (c *CertManager) GetCaFile() string {
 	return c.GetFilePath(RootCA)
 }
@@ -162,6 +167,7 @@ func (c *CertManager) updateCerts(secret *corev1.Secret) {
 		if err := c.createOrUpdateFile(c.GetFilePath(RootCA), ca); err != nil {
 			klog.Errorf("could not update ca, %v", err)
 		}
+		c.caData = ca
 	}
 
 	if cook {

pkg/yurthub/yurtcoordinator/coordinator.go

@@ -105,8 +105,8 @@ type coordinator struct {
 	statusInfoChan    chan statusInfo
 	isPoolCacheSynced bool
 	certMgr           *certmanager.CertManager
-	// cloudCAFilePath is the file path of cloud kubernetes cluster CA cert.
-	cloudCAFilePath string
+	// cloudCAFileData is the file data of cloud kubernetes cluster CA cert.
+	cloudCAFileData []byte
 	// cloudHealthChecker is health checker of cloud APIServers. It is used to
 	// pick a healthy cloud APIServer to proxy heartbeats.
 	cloudHealthChecker   healthchecker.MultipleBackendsHealthChecker
@@ -153,7 +153,7 @@ func NewCoordinator(
 
 	coordinator := &coordinator{
 		ctx:                ctx,
-		cloudCAFilePath:    cfg.CertManager.GetCaFile(),
+		cloudCAFileData:    cfg.CertManager.GetCAData(),
 		cloudHealthChecker: cloudHealthChecker,
 		etcdStorageCfg:     etcdStorageCfg,
 		restConfigMgr:      restMgr,
@@ -425,7 +425,7 @@ func (coordinator *coordinator) newNodeLeaseProxyClient() (coordclientset.LeaseI
 	restCfg := &rest.Config{
 		Host: healthyCloudServer.String(),
 		TLSClientConfig: rest.TLSClientConfig{
-			CAFile:   coordinator.cloudCAFilePath,
+			CAData:   coordinator.cloudCAFileData,
 			CertFile: coordinator.certMgr.GetFilePath(certmanager.NodeLeaseProxyClientCert),
 			KeyFile:  coordinator.certMgr.GetFilePath(certmanager.NodeLeaseProxyClientKey),
 		},