improve ca data for yurthub component

pkg/util/certmanager/pki.go

@@ -84,7 +84,7 @@
 		// load the root ca from the given kubeconfig file
 		config, err := clientcmd.LoadFromFile(kubeConfig)
 		if err != nil || config == nil {
 			return nil, fmt.Errorf("could not load the kubeconfig file(%s), %w",
 				kubeConfig, err)
 		}
 
@@ -173,7 +173,7 @@
 		if os.IsNotExist(err) {
 			return nil, fmt.Errorf("CA file(%s) doesn't exist", caFile)
 		}
 		return nil, fmt.Errorf("could not stat the CA file(%s): %w", caFile, err)
 	}
 
 	caData, err := os.ReadFile(caFile)
@@ -185,3 +185,10 @@
 	certPool.AppendCertsFromPEM(caData)
 	return certPool, nil
 }
+
+// GenCertPoolUseCAData generates a x509 CertPool based on the given CA data
+func GenCertPoolUseCAData(caData []byte) (*x509.CertPool, error) {
+	certPool := x509.NewCertPool()
+	certPool.AppendCertsFromPEM(caData)
+	return certPool, nil
+}

pkg/yurthub/certificate/interfaces.go

@@ -45,6 +45,7 @@ type YurtClientCertificateManager interface {
 	Stop()
 	UpdateBootstrapConf(joinToken string) error
 	GetHubConfFile() string
+	GetCAData() []byte
 	GetCaFile() string
 	GetAPIServerClientCert() *tls.Certificate
 }

pkg/yurthub/certificate/kubeletcertificate/kubelet_certificate.go

@@ -21,6 +21,7 @@
 	"crypto/x509"
 	"errors"
 	"fmt"
+	"os"
 	"time"
 
 	"k8s.io/klog/v2"
@@ -40,6 +41,7 @@
 	kubeletCAFile  string
 	kubeletPemFile string
 	cert           *tls.Certificate
+	caData         []byte
 }
 
 func NewKubeletCertManager(kubeConfFile, kubeletCAFile, kubeletPemFile string) (certificate.YurtClientCertificateManager, error) {
@@ -50,6 +52,10 @@
 	if exist, _ := util.FileExists(kubeletCAFile); !exist {
 		return nil, KubeletCANotExistErr
 	}
+	caData, err := os.ReadFile(kubeletCAFile)
+	if err != nil {
+		return nil, err
+	}
 
 	if exist, _ := util.FileExists(kubeletPemFile); !exist {
 		return nil, KubeletPemNotExistErr
@@ -65,6 +71,7 @@
 		kubeletCAFile:  kubeletCAFile,
 		kubeletPemFile: kubeletPemFile,
 		cert:           cert,
+		caData:         caData,
 	}, nil
 }
 
@@ -84,6 +91,10 @@
 	return kcm.kubeConfFile
 }
 
+func (kcm *kubeletCertManager) GetCAData() []byte {
+	return kcm.caData
+}
+
 func (kcm *kubeletCertManager) GetCaFile() string {
 	return kcm.kubeletCAFile
 }
@@ -96,7 +107,7 @@
 	klog.Warningf("current certificate: %s is expired, reload it", kcm.kubeletPemFile)
 	cert, err := loadFile(kcm.kubeletPemFile)
 	if err != nil {
 		klog.Errorf("could not load client certificate(%s), %v", kcm.kubeletPemFile, err)
 		return nil
 	}
 	kcm.cert = cert

pkg/yurthub/certificate/manager/manager.go

@@ -32,7 +32,6 @@
 	"github.com/openyurtio/openyurt/pkg/yurthub/certificate/kubeletcertificate"
 	hubServerCert "github.com/openyurtio/openyurt/pkg/yurthub/certificate/server"
 	"github.com/openyurtio/openyurt/pkg/yurthub/certificate/token"
-	"github.com/openyurtio/openyurt/pkg/yurthub/util"
 )
 
 const (
@@ -123,12 +122,8 @@
 		errs = append(errs, apiServerClientCertNotReadyError)
 	}
 
-	if exist, err := util.FileExists(hcm.YurtClientCertificateManager.GetCaFile()); !exist {
-		if err == nil {
-			errs = append(errs, caCertIsNotReadyError)
-		} else {
-			errs = append(errs, err)
-		}
+	if len(hcm.YurtClientCertificateManager.GetCAData()) == 0 {
+		errs = append(errs, caCertIsNotReadyError)
 	}
 
 	if hcm.GetHubServerCert() == nil {

pkg/yurthub/certificate/manager/manager_test.go

@@ -24,16 +24,11 @@ import (
 	"testing"
 	"time"
 
-	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/util/wait"
-	"k8s.io/client-go/tools/clientcmd"
-	certutil "k8s.io/client-go/util/cert"
 
 	"github.com/openyurtio/openyurt/cmd/yurthub/app/options"
 	"github.com/openyurtio/openyurt/pkg/projectinfo"
-	kubeconfigutil "github.com/openyurtio/openyurt/pkg/util/kubeconfig"
 	"github.com/openyurtio/openyurt/pkg/yurthub/certificate/testdata"
-	"github.com/openyurtio/openyurt/pkg/yurthub/util"
 )
 
 func TestGetHubServerCertFile(t *testing.T) {
@@ -108,33 +103,6 @@ func TestReady(t *testing.T) {
 		if mgr.Ready() {
 			return true, nil
 		}
-
-		if exist, err := util.FileExists(mgr.GetCaFile()); !exist {
-			if err != nil {
-				t.Logf("could not get ca file(%s), %v", mgr.GetCaFile(), err)
-				return false, err
-			}
-
-			if exist, err := util.FileExists(mgr.GetHubConfFile()); err != nil {
-				t.Logf("could not get hub conf file(%s), %v", mgr.GetHubConfFile(), err)
-				return false, nil
-			} else if exist {
-				t.Logf("%s file already exists, so use it to create ca file", mgr.GetHubConfFile())
-				hubKubeConfig, err := clientcmd.LoadFromFile(mgr.GetHubConfFile())
-				if err != nil {
-					return false, err
-				}
-
-				cluster := kubeconfigutil.GetClusterFromKubeConfig(hubKubeConfig)
-				if cluster != nil {
-					if err := certutil.WriteCert(mgr.GetCaFile(), cluster.CertificateAuthorityData); err != nil {
-						return false, errors.Wrap(err, "couldn't save the CA certificate to disk")
-					}
-				} else {
-					return false, errors.Errorf("couldn't prepare ca.crt(%s) file", mgr.GetCaFile())
-				}
-			}
-		}
 		return false, nil
 	})
 
@@ -161,8 +129,14 @@ func TestReady(t *testing.T) {
 		return
 	}
 	newMgr.Start()
-	if !newMgr.Ready() {
-		t.Errorf("certificates can not be reused")
+	err = wait.PollImmediate(2*time.Second, 1*time.Minute, func() (done bool, err error) {
+		if mgr.Ready() {
+			return true, nil
+		}
+		return false, nil
+	})
+	if err != nil {
+		t.Errorf("certificates are not reused, %v", err)
 	}
 	newMgr.Stop()
 

pkg/yurthub/certificate/token/token.go

@@ -80,6 +80,7 @@
 	joinToken                  string
 	bootstrapFile              string
 	dialer                     *util.Dialer
+	caData                     []byte
 }
 
 // NewYurtHubClientCertManager new a YurtCertificateManager instance
@@ -149,7 +150,7 @@
 func (ycm *yurtHubClientCertManager) Start() {
 	err := ycm.prepareConfigAndCaFile()
 	if err != nil {
 		klog.Errorf("could not prepare config and ca file, %v", err)
 		return
 	}
 
@@ -171,7 +172,7 @@
 	if len(ycm.bootstrapFile) != 0 {
 		// 1. load bootstrap config
 		if tlsBootstrapCfg, err = clientcmd.LoadFromFile(ycm.getBootstrapConfFile()); err != nil {
 			klog.Errorf("maybe hub agent restarted, could not load bootstrap config file(%s), %v.", ycm.getBootstrapConfFile(), err)
 		} else {
 			klog.V(2).Infof("%s file is configured, just use it", ycm.getBootstrapConfFile())
 		}
@@ -203,11 +204,17 @@
 				if err := certutil.WriteCert(ycm.GetCaFile(), cluster.CertificateAuthorityData); err != nil {
 					return errors.Wrap(err, "couldn't save the CA certificate to disk")
 				}
+				ycm.caData = cluster.CertificateAuthorityData
 			} else {
 				return errors.Errorf("couldn't prepare ca.crt(%s) file", ycm.GetCaFile())
 			}
 		} else {
 			klog.V(2).Infof("%s file already exists, so reuse it", ycm.GetCaFile())
+			caData, err := os.ReadFile(ycm.GetCaFile())
+			if err != nil {
+				return err
+			}
+			ycm.caData = caData
 		}
 		return nil
 	}
@@ -220,7 +227,7 @@
 		return errors.Wrap(err, "couldn't stat bootstrap config file")
 	} else if !exist {
 		if tlsBootstrapCfg, err = ycm.retrieveHubBootstrapConfig(ycm.joinToken); err != nil {
 			return errors.Wrap(err, "could not retrieve bootstrap config")
 		}
 	} else {
 		klog.V(2).Infof("%s file already exists, so reuse it", ycm.getBootstrapConfFile())
@@ -250,11 +257,17 @@
 			if err := certutil.WriteCert(ycm.GetCaFile(), cluster.CertificateAuthorityData); err != nil {
 				return errors.Wrap(err, "couldn't save the CA certificate to disk")
 			}
+			ycm.caData = cluster.CertificateAuthorityData
 		} else {
 			return errors.Errorf("couldn't prepare ca.crt(%s) file", ycm.GetCaFile())
 		}
 	} else {
 		klog.V(2).Infof("%s file already exists, so reuse it", ycm.GetCaFile())
+		caData, err := os.ReadFile(ycm.GetCaFile())
+		if err != nil {
+			return err
+		}
+		ycm.caData = caData
 	}
 
 	return nil
@@ -284,6 +297,10 @@
 	return filepath.Join(ycm.hubRunDir, bootstrapConfigFileName)
 }
 
+func (ycm *yurtHubClientCertManager) GetCAData() []byte {
+	return ycm.caData
+}
+
 // GetCaFile returns the path of ca file
 func (ycm *yurtHubClientCertManager) GetCaFile() string {
 	return filepath.Join(ycm.getPkiDir(), hubCaFileName)

pkg/yurthub/transport/transport.go

@@ -34,8 +34,8 @@ type CertGetter interface {
 	// GetAPIServerClientCert returns the currently selected certificate, as well as
 	// the associated certificate and key data in PEM format.
 	GetAPIServerClientCert() *tls.Certificate
-	// Return CA file path.
-	GetCaFile() string
+	// GetCAData returns CA file data.
+	GetCAData() []byte
 }
 
 // Interface is an transport interface for managing clients that used to connecting kube-apiserver
@@ -60,13 +60,12 @@ type transportManager struct {
 
 // NewTransportManager create a transport interface object.
 func NewTransportManager(certGetter CertGetter, stopCh <-chan struct{}) (Interface, error) {
-	caFile := certGetter.GetCaFile()
-	if len(caFile) == 0 {
-		return nil, fmt.Errorf("ca cert file was not prepared when new transport")
+	caData := certGetter.GetCAData()
+	if len(caData) == 0 {
+		return nil, fmt.Errorf("ca cert data was not prepared when new transport")
 	}
-	klog.V(2).Infof("use %s ca cert file to access remote server", caFile)
 
-	cfg, err := tlsConfig(certGetter.GetAPIServerClientCert, caFile)
+	cfg, err := tlsConfig(certGetter.GetAPIServerClientCert, caData)
 	if err != nil {
 		klog.Errorf("could not get tls config when new transport, %v", err)
 		return nil, err
@@ -81,7 +80,7 @@ func NewTransportManager(certGetter CertGetter, stopCh <-chan struct{}) (Interfa
 		DialContext:         d.DialContext,
 	})
 
-	bearerTLSCfg, err := tlsConfig(nil, caFile)
+	bearerTLSCfg, err := tlsConfig(nil, caData)
 	if err != nil {
 		klog.Errorf("could not get tls config when new bearer transport, %v", err)
 		return nil, err
@@ -151,9 +150,9 @@ func (tm *transportManager) start() {
 	}, 10*time.Second, tm.stopCh)
 }
 
-func tlsConfig(current func() *tls.Certificate, caFile string) (*tls.Config, error) {
+func tlsConfig(current func() *tls.Certificate, caData []byte) (*tls.Config, error) {
 	// generate the TLS configuration based on the latest certificate
-	rootCert, err := certmanager.GenCertPoolUseCA(caFile)
+	rootCert, err := certmanager.GenCertPoolUseCAData(caData)
 	if err != nil {
 		klog.Errorf("could not generate a x509 CertPool based on the given CA file, %v", err)
 		return nil, err

pkg/yurthub/yurtcoordinator/certmanager/certmanager.go

@@ -57,7 +57,7 @@
 func NewCertManager(pkiDir, yurtHubNs string, yurtClient kubernetes.Interface, informerFactory informers.SharedInformerFactory) (*CertManager, error) {
 	store := fs.FileSystemOperator{}
 	if err := store.CreateDir(pkiDir); err != nil && err != fs.ErrExists {
 		return nil, fmt.Errorf("could not create dir %s, %v", pkiDir, err)
 	}
 
 	certMgr := &CertManager{
@@ -98,6 +98,7 @@
 	coordinatorCert    *tls.Certificate
 	nodeLeaseProxyCert *tls.Certificate
 	store              fs.FileSystemOperator
+	caData             []byte
 
 	// Used for unit test.
 	secret *corev1.Secret
@@ -115,6 +116,10 @@
 	return c.nodeLeaseProxyCert
 }
 
+func (c *CertManager) GetCAData() []byte {
+	return c.caData
+}
+
 func (c *CertManager) GetCaFile() string {
 	return c.GetFilePath(RootCA)
 }
@@ -139,7 +144,7 @@
 	var coordinatorCert, nodeLeaseProxyCert *tls.Certificate
 	if cook {
 		if cert, err := tls.X509KeyPair(coordinatorClientCrt, coordinatorClientKey); err != nil {
 			klog.Errorf("could not create tls certificate for coordinator, %v", err)
 		} else {
 			coordinatorCert = &cert
 		}
@@ -147,7 +152,7 @@
 
 	if nook {
 		if cert, err := tls.X509KeyPair(nodeLeaseProxyClientCrt, nodeLeaseProxyClientKey); err != nil {
 			klog.Errorf("could not create tls certificate for node lease proxy, %v", err)
 		} else {
 			nodeLeaseProxyCert = &cert
 		}
@@ -160,27 +165,28 @@
 	if caok {
 		klog.Infof("updating coordinator ca cert")
 		if err := c.createOrUpdateFile(c.GetFilePath(RootCA), ca); err != nil {
 			klog.Errorf("could not update ca, %v", err)
 		}
+		c.caData = ca
 	}
 
 	if cook {
 		klog.Infof("updating yurt-coordinator-yurthub client cert and key")
 		if err := c.createOrUpdateFile(c.GetFilePath(YurthubClientKey), coordinatorClientKey); err != nil {
 			klog.Errorf("could not update coordinator client key, %v", err)
 		}
 		if err := c.createOrUpdateFile(c.GetFilePath(YurthubClientCert), coordinatorClientCrt); err != nil {
 			klog.Errorf("could not update coordinator client cert, %v", err)
 		}
 	}
 
 	if nook {
 		klog.Infof("updating node-lease-proxy-client cert and key")
 		if err := c.createOrUpdateFile(c.GetFilePath(NodeLeaseProxyClientKey), nodeLeaseProxyClientKey); err != nil {
 			klog.Errorf("could not update node lease proxy client key, %v", err)
 		}
 		if err := c.createOrUpdateFile(c.GetFilePath(NodeLeaseProxyClientCert), nodeLeaseProxyClientCrt); err != nil {
 			klog.Errorf("could not update node lease proxy client cert, %v", err)
 		}
 	}
 
@@ -199,7 +205,7 @@
 func (c *CertManager) createOrUpdateFile(path string, data []byte) error {
 	if err := c.store.Write(path, data); err == fs.ErrNotExists {
 		if err := c.store.CreateFile(path, data); err != nil {
 			return fmt.Errorf("could not create file at %s, %v", path, err)
 		}
 	} else if err != nil {
 		return fmt.Errorf("could not update file at %s, %v", path, err)

pkg/yurthub/yurtcoordinator/coordinator.go

@@ -105,8 +105,8 @@
 	statusInfoChan    chan statusInfo
 	isPoolCacheSynced bool
 	certMgr           *certmanager.CertManager
-	// cloudCAFilePath is the file path of cloud kubernetes cluster CA cert.
-	cloudCAFilePath string
+	// cloudCAFileData is the file data of cloud kubernetes cluster CA cert.
+	cloudCAFileData []byte
 	// cloudHealthChecker is health checker of cloud APIServers. It is used to
 	// pick a healthy cloud APIServer to proxy heartbeats.
 	cloudHealthChecker   healthchecker.MultipleBackendsHealthChecker
@@ -148,12 +148,12 @@
 	}
 	coordinatorClient, err := kubernetes.NewForConfig(coordinatorRESTCfg)
 	if err != nil {
 		return nil, fmt.Errorf("could not create client for yurt coordinator, %v", err)
 	}
 
 	coordinator := &coordinator{
 		ctx:                ctx,
-		cloudCAFilePath:    cfg.CertManager.GetCaFile(),
+		cloudCAFileData:    cfg.CertManager.GetCAData(),
 		cloudHealthChecker: cloudHealthChecker,
 		etcdStorageCfg:     etcdStorageCfg,
 		restConfigMgr:      restMgr,
@@ -188,7 +188,7 @@
 
 	proxiedClient, err := buildProxiedClientWithUserAgent(fmt.Sprintf("http://%s", cfg.YurtHubProxyServerAddr), constants.DefaultPoolScopedUserAgent)
 	if err != nil {
 		return nil, fmt.Errorf("could not create proxied client, %v", err)
 	}
 
 	// init pool scope resources
@@ -196,7 +196,7 @@
 
 	dynamicClient, err := buildDynamicClientWithUserAgent(fmt.Sprintf("http://%s", cfg.YurtHubProxyServerAddr), constants.DefaultPoolScopedUserAgent)
 	if err != nil {
 		return nil, fmt.Errorf("could not create dynamic client, %v", err)
 	}
 
 	poolScopedCacheSyncManager := &poolScopedCacheSyncManager{
@@ -283,13 +283,13 @@
 			case LeaderHub:
 				poolCacheManager, etcdStorage, cancelEtcdStorage, err = coordinator.buildPoolCacheStore()
 				if err != nil {
 					klog.Errorf("could not create pool scoped cache store and manager, %v", err)
 					coordinator.statusInfoChan <- electorStatusInfo
 					continue
 				}
 
 				if err := coordinator.poolCacheSyncManager.EnsureStart(); err != nil {
 					klog.Errorf("could not sync pool-scoped resource, %v", err)
 					cancelEtcdStorage()
 					coordinator.statusInfoChan <- electorStatusInfo
 					continue
@@ -320,7 +320,7 @@
 
 				if coordinator.needUploadLocalCache {
 					if err := coordinator.uploadLocalCache(etcdStorage); err != nil {
 						klog.Errorf("could not upload local cache when yurthub becomes leader, %v", err)
 					} else {
 						needUploadLocalCache = false
 					}
@@ -328,7 +328,7 @@
 			case FollowerHub:
 				poolCacheManager, etcdStorage, cancelEtcdStorage, err = coordinator.buildPoolCacheStore()
 				if err != nil {
 					klog.Errorf("could not create pool scoped cache store and manager, %v", err)
 					coordinator.statusInfoChan <- electorStatusInfo
 					continue
 				}
@@ -339,7 +339,7 @@
 
 				if coordinator.needUploadLocalCache {
 					if err := coordinator.uploadLocalCache(etcdStorage); err != nil {
 						klog.Errorf("could not upload local cache when yurthub becomes follower, %v", err)
 					} else {
 						needUploadLocalCache = false
 					}
@@ -400,7 +400,7 @@
 	etcdStore, err := etcd.NewStorage(ctx, coordinator.etcdStorageCfg)
 	if err != nil {
 		cancel()
 		return nil, nil, nil, fmt.Errorf("could not create etcd storage, %v", err)
 	}
 	poolCacheManager := cachemanager.NewCacheManager(
 		cachemanager.NewStorageWrapper(etcdStore),
@@ -418,14 +418,14 @@
 func (coordinator *coordinator) newNodeLeaseProxyClient() (coordclientset.LeaseInterface, error) {
 	healthyCloudServer, err := coordinator.cloudHealthChecker.PickHealthyServer()
 	if err != nil {
 		return nil, fmt.Errorf("could not get a healthy cloud APIServer, %v", err)
 	} else if healthyCloudServer == nil {
 		return nil, fmt.Errorf("could not get a healthy cloud APIServer, all server are unhealthy")
 	}
 	restCfg := &rest.Config{
 		Host: healthyCloudServer.String(),
 		TLSClientConfig: rest.TLSClientConfig{
-			CAFile:   coordinator.cloudCAFilePath,
+			CAData:   coordinator.cloudCAFileData,
 			CertFile: coordinator.certMgr.GetFilePath(certmanager.NodeLeaseProxyClientCert),
 			KeyFile:  coordinator.certMgr.GetFilePath(certmanager.NodeLeaseProxyClientKey),
 		},
@@ -433,7 +433,7 @@
 	}
 	cloudClient, err := kubernetes.NewForConfig(restCfg)
 	if err != nil {
 		return nil, fmt.Errorf("could not create cloud client, %v", err)
 	}
 
 	return cloudClient.CoordinationV1().Leases(corev1.NamespaceNodeLease), nil
@@ -458,7 +458,7 @@
 		cloudLease, err := cloudLeaseClient.Get(coordinator.ctx, newLease.Name, metav1.GetOptions{})
 		if apierrors.IsNotFound(err) {
 			if _, err := cloudLeaseClient.Create(coordinator.ctx, cloudLease, metav1.CreateOptions{}); err != nil {
 				klog.Errorf("could not create lease %s at cloud, %v", newLease.Name, err)
 				continue
 			}
 		}
@@ -466,7 +466,7 @@
 		cloudLease.Annotations = newLease.Annotations
 		cloudLease.Spec.RenewTime = newLease.Spec.RenewTime
 		if updatedLease, err := cloudLeaseClient.Update(coordinator.ctx, cloudLease, metav1.UpdateOptions{}); err != nil {
 			klog.Errorf("could not update lease %s at cloud, %v", newLease.Name, err)
 			continue
 		} else {
 			klog.V(2).Infof("delegate node lease for %s", updatedLease.Name)
@@ -498,7 +498,7 @@
 	if !p.isRunning {
 		err := p.coordinatorClient.CoordinationV1().Leases(namespaceInformerLease).Delete(p.ctx, nameInformerLease, metav1.DeleteOptions{})
 		if err != nil && !apierrors.IsNotFound(err) {
 			return fmt.Errorf("could not delete informer sync lease, %v", err)
 		}
 
 		etcdStore := p.getEtcdStore()
@@ -506,7 +506,7 @@
 			return fmt.Errorf("got empty etcd storage")
 		}
 		if err := etcdStore.DeleteComponentResources(constants.DefaultPoolScopedUserAgent); err != nil {
 			return fmt.Errorf("could not clean old pool-scoped cache, %v", err)
 		}
 
 		ctx, cancel := context.WithCancel(p.ctx)
@@ -546,7 +546,7 @@
 		p.renewInformerLease(ctx, informerLease)
 		return
 	}
 	klog.Error("could not wait for cache synced, it was canceled")
 }
 
 func (p *poolScopedCacheSyncManager) renewInformerLease(ctx context.Context, lease informerLease) {
@@ -559,7 +559,7 @@
 		case <-t.C:
 			newLease, err := lease.Update(p.informerSyncedLease)
 			if err != nil {
 				klog.Errorf("could not update informer lease, %v", err)
 				continue
 			}
 			p.informerSyncedLease = newLease
@@ -613,12 +613,12 @@
 	for k, b := range objBytes {
 		rv, err := getRv(b)
 		if err != nil {
 			klog.Errorf("could not get name from bytes %s, %v", string(b), err)
 			continue
 		}
 
 		if err := l.createOrUpdate(k, b, rv); err != nil {
 			klog.Errorf("could not upload %s, %v", k.Key(), err)
 		}
 	}
 }
@@ -648,25 +648,25 @@
 		}
 		localKeys, err := l.diskStorage.ListResourceKeysOfComponent(info.Component, gvr)
 		if err != nil {
 			klog.Errorf("could not get object keys from disk for %s, %v", gvr.String(), err)
 			continue
 		}
 
 		for _, k := range localKeys {
 			buf, err := l.diskStorage.Get(k)
 			if err != nil {
 				klog.Errorf("could not read local cache of key %s, %v", k.Key(), err)
 				continue
 			}
 			buildInfo, err := disk.ExtractKeyBuildInfo(k)
 			if err != nil {
 				klog.Errorf("could not extract key build info from local cache of key %s, %v", k.Key(), err)
 				continue
 			}
 
 			poolCacheKey, err := l.etcdStorage.KeyFunc(*buildInfo)
 			if err != nil {
 				klog.Errorf("could not generate pool cache key from local cache key %s, %v", k.Key(), err)
 				continue
 			}
 			objBytes[poolCacheKey] = buf
@@ -749,12 +749,12 @@
 func getRv(objBytes []byte) (uint64, error) {
 	obj := &unstructured.Unstructured{}
 	if err := json.Unmarshal(objBytes, obj); err != nil {
 		return 0, fmt.Errorf("could not unmarshal json: %v", err)
 	}
 
 	rv, err := strconv.ParseUint(obj.GetResourceVersion(), 10, 64)
 	if err != nil {
 		return 0, fmt.Errorf("could not parse rv %s of pod %s, %v", obj.GetName(), obj.GetResourceVersion(), err)
 	}
 
 	return rv, nil