From 916f2262871410f71372827751132518ae07db27 Mon Sep 17 00:00:00 2001 From: qclc <39878907+qclc@users.noreply.github.com> Date: Tue, 30 Nov 2021 15:18:40 +0800 Subject: [PATCH] delete the kubelet cert-mgr-mode (#639) --- cmd/yurthub/app/options/options.go | 2 +- cmd/yurthub/app/start.go | 2 - pkg/yurthub/certificate/kubelet/cert_mgr.go | 180 ------------------ .../certificate/kubelet/cert_mgr_test.go | 166 ---------------- pkg/yurthub/kubernetes/rest/config.go | 41 +--- pkg/yurthub/kubernetes/rest/config_test.go | 7 +- pkg/yurthub/util/util.go | 4 +- 7 files changed, 11 insertions(+), 391 deletions(-) delete mode 100644 pkg/yurthub/certificate/kubelet/cert_mgr.go delete mode 100644 pkg/yurthub/certificate/kubelet/cert_mgr_test.go diff --git a/cmd/yurthub/app/options/options.go b/cmd/yurthub/app/options/options.go index cef7f7ae880..b11c3c6bc93 100644 --- a/cmd/yurthub/app/options/options.go +++ b/cmd/yurthub/app/options/options.go @@ -135,7 +135,7 @@ func (o *YurtHubOptions) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&o.YurtHubProxyPort, "proxy-port", o.YurtHubProxyPort, "the port on which to proxy HTTP requests to kube-apiserver") fs.StringVar(&o.YurtHubProxySecurePort, "proxy-secure-port", o.YurtHubProxySecurePort, "the port on which to proxy HTTPS requests to kube-apiserver") fs.StringVar(&o.ServerAddr, "server-addr", o.ServerAddr, "the address of Kubernetes kube-apiserver,the format is: \"server1,server2,...\"") - fs.StringVar(&o.CertMgrMode, "cert-mgr-mode", o.CertMgrMode, "the cert manager mode, kubelet: use certificates that belongs to kubelet, hubself: auto generate client cert for hub agent.") + fs.StringVar(&o.CertMgrMode, "cert-mgr-mode", o.CertMgrMode, "the cert manager mode, hubself: auto generate client cert for hub agent.") fs.StringVar(&o.KubeletRootCAFilePath, "kubelet-ca-file", o.KubeletRootCAFilePath, "the ca file path used by kubelet.") fs.StringVar(&o.KubeletPairFilePath, "kubelet-client-certificate", o.KubeletPairFilePath, "the path of kubelet client certificate file.") fs.IntVar(&o.GCFrequency, "gc-frequency", o.GCFrequency, "the frequency to gc cache in storage(unit: minute).") diff --git a/cmd/yurthub/app/start.go b/cmd/yurthub/app/start.go index c4c5a972cf1..0af9ed9d799 100644 --- a/cmd/yurthub/app/start.go +++ b/cmd/yurthub/app/start.go @@ -26,7 +26,6 @@ import ( "github.com/openyurtio/openyurt/pkg/yurthub/cachemanager" "github.com/openyurtio/openyurt/pkg/yurthub/certificate" "github.com/openyurtio/openyurt/pkg/yurthub/certificate/hubself" - "github.com/openyurtio/openyurt/pkg/yurthub/certificate/kubelet" "github.com/openyurtio/openyurt/pkg/yurthub/gc" "github.com/openyurtio/openyurt/pkg/yurthub/healthchecker" "github.com/openyurtio/openyurt/pkg/yurthub/kubernetes/rest" @@ -84,7 +83,6 @@ func Run(cfg *config.YurtHubConfiguration, stopCh <-chan struct{}) error { trace := 1 klog.Infof("%d. register cert managers", trace) cmr := certificate.NewCertificateManagerRegistry() - kubelet.Register(cmr) hubself.Register(cmr) trace++ diff --git a/pkg/yurthub/certificate/kubelet/cert_mgr.go b/pkg/yurthub/certificate/kubelet/cert_mgr.go deleted file mode 100644 index 1c53331fe8c..00000000000 --- a/pkg/yurthub/certificate/kubelet/cert_mgr.go +++ /dev/null @@ -1,180 +0,0 @@ -/* -Copyright 2020 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubelet - -import ( - "crypto/tls" - "crypto/x509" - "fmt" - "net/url" - "sync" - "time" - - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/klog" - - "github.com/openyurtio/openyurt/cmd/yurthub/app/config" - "github.com/openyurtio/openyurt/pkg/yurthub/certificate" - "github.com/openyurtio/openyurt/pkg/yurthub/certificate/interfaces" - "github.com/openyurtio/openyurt/pkg/yurthub/util" -) - -const ( - certVerifyDuration = 30 * time.Minute -) - -// Register registers a YurtCertificateManager -func Register(cmr *certificate.CertificateManagerRegistry) { - cmr.Register(util.KubeletCertificateManagerName, func(cfg *config.YurtHubConfiguration) (interfaces.YurtCertificateManager, error) { - return NewKubeletCertManager(cfg, 0) - }) -} - -type kubeletCertManager struct { - certAccessLock sync.RWMutex - pairFile string - cert *tls.Certificate - stopCh chan struct{} - remoteServers []*url.URL - caFile string - certVerifyDuration time.Duration - stopped bool -} - -// NewKubeletCertManager creates a YurtCertificateManager -func NewKubeletCertManager(cfg *config.YurtHubConfiguration, period time.Duration) (interfaces.YurtCertificateManager, error) { - var cert *tls.Certificate - pairFile := cfg.KubeletPairFilePath - if cfg == nil || len(cfg.RemoteServers) == 0 { - return nil, fmt.Errorf("hub configuration is invalid") - } - - if period == time.Duration(0) { - period = certVerifyDuration - } - - if pairFileExists, err := util.FileExists(pairFile); err != nil { - return nil, err - } else if pairFileExists { - klog.Infof("Loading cert/key pair from %q.", pairFile) - cert, err = loadFile(pairFile) - if err != nil { - return nil, err - } - } - - return &kubeletCertManager{ - pairFile: pairFile, - cert: cert, - remoteServers: cfg.RemoteServers, - caFile: cfg.KubeletRootCAFilePath, - certVerifyDuration: period, - stopCh: make(chan struct{}), - }, nil -} - -// Stop stop cert manager -func (kcm *kubeletCertManager) Stop() { - kcm.certAccessLock.Lock() - defer kcm.certAccessLock.Unlock() - if kcm.stopped { - return - } - close(kcm.stopCh) - kcm.stopped = true -} - -// Start runs certificate reloading after rotation -func (kcm *kubeletCertManager) Start() { - go wait.Until(func() { - newCert, err := loadFile(kcm.pairFile) - if err != nil { - klog.Errorf("failed to load cert file %s, %v", kcm.pairFile, err) - return - } - - certChanged := true - kcm.certAccessLock.RLock() - if kcm.cert.Leaf.NotAfter.Equal(newCert.Leaf.NotAfter) { - certChanged = false - } - kcm.certAccessLock.RUnlock() - - if certChanged { - klog.Infof("cert file %s is updated", kcm.pairFile) - kcm.updateCert(newCert) - } - - }, kcm.certVerifyDuration, kcm.stopCh) -} - -// Current get the current certificate -func (kcm *kubeletCertManager) Current() *tls.Certificate { - kcm.certAccessLock.RLock() - defer kcm.certAccessLock.RUnlock() - return kcm.cert -} - -// ServerHealthy always returns true -func (kcm *kubeletCertManager) ServerHealthy() bool { - return true -} - -// GetCaFile get a ca file -func (kcm *kubeletCertManager) GetCaFile() string { - return kcm.caFile -} - -// GetConfFilePath get a kube-config file path, but the kubelet mode just using the ca and pair, so return empty -func (kcm *kubeletCertManager) GetConfFilePath() string { - return "" -} - -func (kcm *kubeletCertManager) NotExpired() bool { - kcm.certAccessLock.RLock() - defer kcm.certAccessLock.RUnlock() - if kcm.cert == nil || kcm.cert.Leaf == nil || time.Now().After(kcm.cert.Leaf.NotAfter) { - klog.V(2).Infof("Current certificate is expired.") - return false - } - return true -} - -func (kcm *kubeletCertManager) updateCert(c *tls.Certificate) { - kcm.certAccessLock.Lock() - defer kcm.certAccessLock.Unlock() - kcm.cert = c -} - -// Update do nothing -func (kcm *kubeletCertManager) Update(_ *config.YurtHubConfiguration) error { - return nil -} - -func loadFile(pairFile string) (*tls.Certificate, error) { - cert, err := tls.LoadX509KeyPair(pairFile, pairFile) - if err != nil { - return nil, fmt.Errorf("could not convert data from %q into cert/key pair: %v", pairFile, err) - } - - certs, err := x509.ParseCertificates(cert.Certificate[0]) - if err != nil { - return nil, fmt.Errorf("unable to parse certificate data: %v", err) - } - cert.Leaf = certs[0] - return &cert, nil -} diff --git a/pkg/yurthub/certificate/kubelet/cert_mgr_test.go b/pkg/yurthub/certificate/kubelet/cert_mgr_test.go deleted file mode 100644 index b70b5ce77a3..00000000000 --- a/pkg/yurthub/certificate/kubelet/cert_mgr_test.go +++ /dev/null @@ -1,166 +0,0 @@ -/* -Copyright 2020 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubelet - -import ( - "bytes" - "crypto/tls" - "crypto/x509" - "fmt" - "io/ioutil" - "net/url" - "os" - "path/filepath" - "testing" - "time" - - "github.com/openyurtio/openyurt/cmd/yurthub/app/config" - "github.com/openyurtio/openyurt/pkg/yurthub/util" -) - -var storeCertData = newCertificateData(`-----BEGIN CERTIFICATE----- -MIICRzCCAfGgAwIBAgIJALMb7ecMIk3MMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNV -BAYTAkdCMQ8wDQYDVQQIDAZMb25kb24xDzANBgNVBAcMBkxvbmRvbjEYMBYGA1UE -CgwPR2xvYmFsIFNlY3VyaXR5MRYwFAYDVQQLDA1JVCBEZXBhcnRtZW50MRswGQYD -VQQDDBJ0ZXN0LWNlcnRpZmljYXRlLTAwIBcNMTcwNDI2MjMyNjUyWhgPMjExNzA0 -MDIyMzI2NTJaMH4xCzAJBgNVBAYTAkdCMQ8wDQYDVQQIDAZMb25kb24xDzANBgNV -BAcMBkxvbmRvbjEYMBYGA1UECgwPR2xvYmFsIFNlY3VyaXR5MRYwFAYDVQQLDA1J -VCBEZXBhcnRtZW50MRswGQYDVQQDDBJ0ZXN0LWNlcnRpZmljYXRlLTAwXDANBgkq -hkiG9w0BAQEFAANLADBIAkEAtBMa7NWpv3BVlKTCPGO/LEsguKqWHBtKzweMY2CV -tAL1rQm913huhxF9w+ai76KQ3MHK5IVnLJjYYA5MzP2H5QIDAQABo1AwTjAdBgNV -HQ4EFgQU22iy8aWkNSxv0nBxFxerfsvnZVMwHwYDVR0jBBgwFoAU22iy8aWkNSxv -0nBxFxerfsvnZVMwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAANBAEOefGbV -NcHxklaW06w6OBYJPwpIhCVozC1qdxGX1dg8VkEKzjOzjgqVD30m59OFmSlBmHsl -nkVA6wyOSDYBf3o= ------END CERTIFICATE-----`, `-----BEGIN RSA PRIVATE KEY----- -MIIBUwIBADANBgkqhkiG9w0BAQEFAASCAT0wggE5AgEAAkEAtBMa7NWpv3BVlKTC -PGO/LEsguKqWHBtKzweMY2CVtAL1rQm913huhxF9w+ai76KQ3MHK5IVnLJjYYA5M -zP2H5QIDAQABAkAS9BfXab3OKpK3bIgNNyp+DQJKrZnTJ4Q+OjsqkpXvNltPJosf -G8GsiKu/vAt4HGqI3eU77NvRI+mL4MnHRmXBAiEA3qM4FAtKSRBbcJzPxxLEUSwg -XSCcosCktbkXvpYrS30CIQDPDxgqlwDEJQ0uKuHkZI38/SPWWqfUmkecwlbpXABK -iQIgZX08DA8VfvcA5/Xj1Zjdey9FVY6POLXen6RPiabE97UCICp6eUW7ht+2jjar -e35EltCRCjoejRHTuN9TC0uCoVipAiAXaJIx/Q47vGwiw6Y8KXsNU6y54gTbOSxX -54LzHNk/+Q== ------END RSA PRIVATE KEY-----`) - -type certificateData struct { - keyPEM []byte - certificatePEM []byte - certificate *tls.Certificate -} - -func newCertificateData(certificatePEM string, keyPEM string) *certificateData { - certificate, err := tls.X509KeyPair([]byte(certificatePEM), []byte(keyPEM)) - if err != nil { - panic(fmt.Sprintf("Unable to initialize certificate: %v", err)) - } - certs, err := x509.ParseCertificates(certificate.Certificate[0]) - if err != nil { - panic(fmt.Sprintf("Unable to initialize certificate leaf: %v", err)) - } - certificate.Leaf = certs[0] - return &certificateData{ - keyPEM: []byte(keyPEM), - certificatePEM: []byte(certificatePEM), - certificate: &certificate, - } -} - -func TestLoadFile(t *testing.T) { - dir, err := ioutil.TempDir("", "k8s-test-load-cert-key-blocks") - if err != nil { - t.Fatalf("Unable to create the test directory %q: %v", dir, err) - } - defer func() { - if err := os.RemoveAll(dir); err != nil { - t.Errorf("Unable to clean up test directory %q: %v", dir, err) - } - }() - - pairFile := filepath.Join(dir, "kubelet-pair.pem") - - tests := []struct { - desc string - data []byte - }{ - {desc: "cert and key", data: bytes.Join([][]byte{storeCertData.certificatePEM, storeCertData.keyPEM}, []byte("\n"))}, - {desc: "key and cert", data: bytes.Join([][]byte{storeCertData.keyPEM, storeCertData.certificatePEM}, []byte("\n"))}, - } - for _, tt := range tests { - t.Run(tt.desc, func(t *testing.T) { - if err := ioutil.WriteFile(pairFile, tt.data, 0600); err != nil { - t.Fatalf("Unable to create the file %q: %v", pairFile, err) - } - cert, err := loadFile(pairFile) - if err != nil { - t.Fatalf("Could not load certificate from disk: %v", err) - } - if cert == nil { - t.Fatalf("There was no error, but no certificate data was returned.") - } - if cert.Leaf == nil { - t.Fatalf("Got an empty leaf, expected private data.") - } - }) - } -} - -func TestCurrent(t *testing.T) { - dir, err := ioutil.TempDir("", "k8s-test-load-cert-key-blocks") - if err != nil { - t.Fatalf("Unable to create the test directory %q: %v", dir, err) - } - defer func() { - if err := os.RemoveAll(dir); err != nil { - t.Errorf("Unable to clean up test directory %q: %v", dir, err) - } - }() - - pairFile := filepath.Join(dir, "kubelet-client-current.pem") - certData := bytes.Join([][]byte{storeCertData.certificatePEM, storeCertData.keyPEM}, []byte("\n")) - - if err := ioutil.WriteFile(pairFile, certData, 0600); err != nil { - t.Fatalf("Unable to create the file %q: %v", pairFile, err) - } - - u, _ := url.Parse("http://127.0.0.1:8080") - cfg := &config.YurtHubConfiguration{ - RemoteServers: []*url.URL{u}, - KubeletPairFilePath: pairFile, - KubeletRootCAFilePath: util.DefaultKubeletRootCAFilePath, - } - // new kubelet cert manager - m, err := NewKubeletCertManager(cfg, 10*time.Second) - if err != nil { - t.Errorf("failed to new kubelet cert manager, %v", err) - } - m.Start() - defer m.Stop() - - // wait over 10s for cert update - time.Sleep(15 * time.Second) - - // verify current cert - cert := m.Current() - if cert == nil { - t.Fatalf("no certificate data was returned") - } - - if cert.Leaf == nil { - t.Errorf("Got an empty leaf, expected private data") - } -} diff --git a/pkg/yurthub/kubernetes/rest/config.go b/pkg/yurthub/kubernetes/rest/config.go index 85a7c30ff71..4ee1e3ac39c 100644 --- a/pkg/yurthub/kubernetes/rest/config.go +++ b/pkg/yurthub/kubernetes/rest/config.go @@ -30,23 +30,19 @@ import ( ) type RestConfigManager struct { - remoteServers []*url.URL - certMgrMode string - kubeletRootCAFilePath string - kubeletPairFilePath string - checker healthchecker.HealthChecker - certManager interfaces.YurtCertificateManager + remoteServers []*url.URL + certMgrMode string + checker healthchecker.HealthChecker + certManager interfaces.YurtCertificateManager } // NewRestConfigManager creates a *RestConfigManager object func NewRestConfigManager(cfg *config.YurtHubConfiguration, certMgr interfaces.YurtCertificateManager, healthChecker healthchecker.HealthChecker) (*RestConfigManager, error) { mgr := &RestConfigManager{ - remoteServers: cfg.RemoteServers, - certMgrMode: cfg.CertMgrMode, - kubeletRootCAFilePath: cfg.KubeletRootCAFilePath, - kubeletPairFilePath: cfg.KubeletPairFilePath, - checker: healthChecker, - certManager: certMgr, + remoteServers: cfg.RemoteServers, + certMgrMode: cfg.CertMgrMode, + checker: healthChecker, + certManager: certMgr, } return mgr, nil } @@ -57,32 +53,11 @@ func (rcm *RestConfigManager) GetRestConfig(needHealthyServer bool) *rest.Config switch certMgrMode { case util.YurtHubCertificateManagerName: return rcm.getHubselfRestConfig(needHealthyServer) - case util.KubeletCertificateManagerName: - return rcm.getKubeletRestConfig(rcm.kubeletRootCAFilePath, rcm.kubeletPairFilePath, needHealthyServer) default: return nil } } -// getKubeletRestConfig gets rest client config from kubelet.conf -func (rcm *RestConfigManager) getKubeletRestConfig(kubeletRootCAFilePath, kubeletPairFilePath string, needHealthyServer bool) *rest.Config { - healthyServer := rcm.remoteServers[0] - if needHealthyServer { - healthyServer = rcm.getHealthyServer() - if healthyServer == nil { - klog.Infof("all of remote servers are unhealthy, so return nil for rest config") - return nil - } - } - - cfg, err := util.LoadKubeletRestClientConfig(healthyServer, kubeletRootCAFilePath, kubeletPairFilePath) - if err != nil { - klog.Errorf("could not load kubelet rest client config, %v", err) - return nil - } - return cfg -} - // getHubselfRestConfig gets rest client config from hub agent conf file. func (rcm *RestConfigManager) getHubselfRestConfig(needHealthyServer bool) *rest.Config { healthyServer := rcm.remoteServers[0] diff --git a/pkg/yurthub/kubernetes/rest/config_test.go b/pkg/yurthub/kubernetes/rest/config_test.go index 8e689f5cc91..fd3307f8a97 100644 --- a/pkg/yurthub/kubernetes/rest/config_test.go +++ b/pkg/yurthub/kubernetes/rest/config_test.go @@ -22,14 +22,12 @@ import ( "os" "path/filepath" "testing" - "time" "k8s.io/client-go/rest" "github.com/openyurtio/openyurt/cmd/yurthub/app/config" "github.com/openyurtio/openyurt/pkg/yurthub/certificate/hubself" "github.com/openyurtio/openyurt/pkg/yurthub/certificate/interfaces" - "github.com/openyurtio/openyurt/pkg/yurthub/certificate/kubelet" "github.com/openyurtio/openyurt/pkg/yurthub/healthchecker" "github.com/openyurtio/openyurt/pkg/yurthub/storage/disk" ) @@ -126,16 +124,13 @@ func TestGetRestConfig(t *testing.T) { desc string mode string }{ - {desc: "kubelet mode", mode: "kubelet"}, {desc: "hubself mode", mode: "hubself"}, } for _, tt := range tests { t.Run(tt.desc, func(t *testing.T) { cfg.CertMgrMode = tt.mode var certMgr interfaces.YurtCertificateManager - if tt.mode == "kubelet" { - certMgr, err = kubelet.NewKubeletCertManager(cfg, 10*time.Second) - } else if tt.mode == "hubself" { + if tt.mode == "hubself" { certMgr, err = hubself.NewFakeYurtHubCertManager(testDir, yurthubCon, string(certificatePEM), string(keyPEM)) certMgr.Start() } diff --git a/pkg/yurthub/util/util.go b/pkg/yurthub/util/util.go index 1456def04cb..8f5b2099623 100644 --- a/pkg/yurthub/util/util.go +++ b/pkg/yurthub/util/util.go @@ -58,8 +58,6 @@ const ( const ( // YurtHubCertificateManagerName represents the certificateManager name in yurthub mode YurtHubCertificateManagerName = "hubself" - // KubeletCertificateManagerName represents the certificateManager name in kubelet mode - KubeletCertificateManagerName = "kubelet" // DefaultKubeletPairFilePath represents the default kubelet pair file path DefaultKubeletPairFilePath = "/var/lib/kubelet/pki/kubelet-client-current.pem" // DefaultKubeletRootCAFilePath represents the default kubelet ca file path @@ -324,7 +322,7 @@ func IsSupportedLBMode(lbMode string) bool { // IsSupportedCertMode check cert mode is supported or not func IsSupportedCertMode(certMode string) bool { switch certMode { - case KubeletCertificateManagerName, YurtHubCertificateManagerName: + case YurtHubCertificateManagerName: return true }