Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve certificate manager #1133

Merged
merged 2 commits into from
Jan 12, 2023
Merged

Improve certificate manager #1133

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c0339f4
improve certificate manager for yurthub
rambohe-ch Jan 10, 2023
f6a8fd7
fix hub root dir comment
rambohe-ch Jan 12, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions cmd/yurt-tunnel-server/app/start.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -177,12 +177,12 @@ func Run(cfg *config.CompletedConfig, stopCh <-chan struct{}) error {
}, stopCh)

// 6. generate the TLS configuration based on the latest certificate
tlsCfg, err := certmanager.GenTLSConfigUseCertMgrAndCertPool(serverCertMgr, cfg.RootCert, "server")
tlsCfg, err := certmanager.GenTLSConfigUseCurrentCertAndCertPool(serverCertMgr.Current, cfg.RootCert, "server")
if err != nil {
return err
}

proxyClientTlsCfg, err := certmanager.GenTLSConfigUseCertMgrAndCertPool(tunnelProxyCertMgr, cfg.RootCert, "client")
proxyClientTlsCfg, err := certmanager.GenTLSConfigUseCurrentCertAndCertPool(tunnelProxyCertMgr.Current, cfg.RootCert, "client")
if err != nil {
return err
}
Expand Down
7 changes: 2 additions & 5 deletions cmd/yurthub/app/config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,9 +65,6 @@ type YurtHubConfiguration struct {
YurtHubProxyServerDummyAddr string
YurtHubProxyServerSecureDummyAddr string
GCFrequency int
CertMgrMode string
KubeletRootCAFilePath string
KubeletPairFilePath string
NodeName string
HeartbeatFailedRetry int
HeartbeatHealthyThreshold int
Expand All @@ -92,6 +89,7 @@ type YurtHubConfiguration struct {
CertIPs []net.IP
CoordinatorServer *url.URL
MinRequestTimeout time.Duration
CaCertHashes []string
}

// Complete converts *options.YurtHubOptions to *YurtHubConfiguration
Expand Down Expand Up @@ -156,8 +154,6 @@ func Complete(options *options.YurtHubOptions) (*YurtHubConfiguration, error) {
YurtHubProxyServerDummyAddr: proxyServerDummyAddr,
YurtHubProxyServerSecureDummyAddr: proxySecureServerDummyAddr,
GCFrequency: options.GCFrequency,
KubeletRootCAFilePath: options.KubeletRootCAFilePath,
KubeletPairFilePath: options.KubeletPairFilePath,
NodeName: options.NodeName,
HeartbeatFailedRetry: options.HeartbeatFailedRetry,
HeartbeatHealthyThreshold: options.HeartbeatHealthyThreshold,
Expand All @@ -180,6 +176,7 @@ func Complete(options *options.YurtHubOptions) (*YurtHubConfiguration, error) {
FilterManager: filterManager,
CertIPs: certIPs,
MinRequestTimeout: options.MinRequestTimeout,
CaCertHashes: options.CACertHashes,
}

return cfg, nil
Expand Down
15 changes: 9 additions & 6 deletions cmd/yurthub/app/options/options.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,6 @@ type YurtHubOptions struct {
YurtHubProxySecurePort string
GCFrequency int
YurtHubCertOrganizations string
KubeletRootCAFilePath string
KubeletPairFilePath string
NodeName string
NodePoolName string
LBMode string
Expand All @@ -74,6 +72,8 @@ type YurtHubOptions struct {
KubeletHealthGracePeriod time.Duration
EnableNodePool bool
MinRequestTimeout time.Duration
CACertHashes []string
UnsafeSkipCAVerification bool
}

// NewYurtHubOptions creates a new YurtHubOptions with a default config.
Expand All @@ -85,8 +85,6 @@ func NewYurtHubOptions() *YurtHubOptions {
YurtHubPort: util.YurtHubPort,
YurtHubProxySecurePort: util.YurtHubProxySecurePort,
GCFrequency: 120,
KubeletRootCAFilePath: util.DefaultKubeletRootCAFilePath,
KubeletPairFilePath: util.DefaultKubeletPairFilePath,
LBMode: "rr",
HeartbeatFailedRetry: 3,
HeartbeatHealthyThreshold: 2,
Expand All @@ -106,6 +104,7 @@ func NewYurtHubOptions() *YurtHubOptions {
KubeletHealthGracePeriod: time.Second * 40,
EnableNodePool: true,
MinRequestTimeout: time.Second * 1800,
UnsafeSkipCAVerification: true,
}
return o
}
Expand All @@ -132,6 +131,10 @@ func (options *YurtHubOptions) Validate() error {
return fmt.Errorf("dummy ip %s is not invalid, %w", options.HubAgentDummyIfIP, err)
}

if len(options.CACertHashes) == 0 && !options.UnsafeSkipCAVerification {
return fmt.Errorf("Set --discovery-token-unsafe-skip-ca-verification flag as true or pass CACertHashes to continue")
}

return nil
}

Expand All @@ -144,8 +147,6 @@ func (o *YurtHubOptions) AddFlags(fs *pflag.FlagSet) {
fs.StringVar(&o.YurtHubProxySecurePort, "proxy-secure-port", o.YurtHubProxySecurePort, "the port on which to proxy HTTPS requests to kube-apiserver")
fs.StringVar(&o.ServerAddr, "server-addr", o.ServerAddr, "the address of Kubernetes kube-apiserver,the format is: \"server1,server2,...\"")
fs.StringVar(&o.YurtHubCertOrganizations, "hub-cert-organizations", o.YurtHubCertOrganizations, "Organizations that will be added into hub's client certificate in hubself cert-mgr-mode, the format is: certOrg1,certOrg1,...")
fs.StringVar(&o.KubeletRootCAFilePath, "kubelet-ca-file", o.KubeletRootCAFilePath, "the ca file path used by kubelet.")
fs.StringVar(&o.KubeletPairFilePath, "kubelet-client-certificate", o.KubeletPairFilePath, "the path of kubelet client certificate file.")
fs.IntVar(&o.GCFrequency, "gc-frequency", o.GCFrequency, "the frequency to gc cache in storage(unit: minute).")
fs.StringVar(&o.NodeName, "node-name", o.NodeName, "the name of node that runs hub agent")
fs.StringVar(&o.LBMode, "lb-mode", o.LBMode, "the mode of load balancer to connect remote servers(rr, priority)")
Expand All @@ -171,6 +172,8 @@ func (o *YurtHubOptions) AddFlags(fs *pflag.FlagSet) {
fs.DurationVar(&o.KubeletHealthGracePeriod, "kubelet-health-grace-period", o.KubeletHealthGracePeriod, "the amount of time which we allow kubelet to be unresponsive before stop renew node lease")
fs.BoolVar(&o.EnableNodePool, "enable-node-pool", o.EnableNodePool, "enable list/watch nodepools resource or not for filters(only used for testing)")
fs.DurationVar(&o.MinRequestTimeout, "min-request-timeout", o.MinRequestTimeout, "An optional field indicating at least how long a proxy handler must keep a request open before timing it out. Currently only honored by the local watch request handler(use request parameter timeoutSeconds firstly), which picks a randomized value above this number as the connection timeout, to spread out load.")
fs.StringSliceVar(&o.CACertHashes, "discovery-token-ca-cert-hash", []string{}, "For token-based discovery, validate that the root CA public key matches this hash (format: \"<type>:<value>\").")
fs.BoolVar(&o.UnsafeSkipCAVerification, "discovery-token-unsafe-skip-ca-verification", o.UnsafeSkipCAVerification, "For token-based discovery, allow joining without --discovery-token-ca-cert-hash pinning.")
}

// verifyDummyIP verify the specified ip is valid or not and set the default ip if empty
Expand Down
21 changes: 9 additions & 12 deletions cmd/yurthub/app/start.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ package app
import (
"fmt"
"net/url"
"path/filepath"
"time"

"github.com/spf13/cobra"
Expand All @@ -33,7 +32,7 @@ import (
"github.com/openyurtio/openyurt/cmd/yurthub/app/options"
"github.com/openyurtio/openyurt/pkg/projectinfo"
"github.com/openyurtio/openyurt/pkg/yurthub/cachemanager"
"github.com/openyurtio/openyurt/pkg/yurthub/certificate/hubself"
"github.com/openyurtio/openyurt/pkg/yurthub/certificate/token"
"github.com/openyurtio/openyurt/pkg/yurthub/gc"
"github.com/openyurtio/openyurt/pkg/yurthub/healthchecker"
hubrest "github.com/openyurtio/openyurt/pkg/yurthub/kubernetes/rest"
Expand Down Expand Up @@ -87,24 +86,23 @@ func NewCmdStartYurtHub(stopCh <-chan struct{}) *cobra.Command {
func Run(cfg *config.YurtHubConfiguration, stopCh <-chan struct{}) error {
trace := 1
klog.Infof("%d. register cert managers", trace)
certManager, err := hubself.NewYurtHubCertManager(cfg)
certManager, err := token.NewYurtHubCertManager(nil, cfg, stopCh)
if err != nil {
return fmt.Errorf("failed to create cert manager for yurthub, %v", err)
}
trace++

certManager.Start()
defer certManager.Stop()
err = wait.PollImmediate(5*time.Second, 4*time.Minute, func() (bool, error) {
curr := certManager.Current()
if curr != nil {
isReady := certManager.Ready()
if isReady {
return true, nil
}

klog.Infof("waiting for preparing client certificate")
return false, nil
})
if err != nil {
return fmt.Errorf("client certificate preparation failed, %v", err)
return fmt.Errorf("hub certificates preparation failed, %v", err)
}
trace++

Expand Down Expand Up @@ -137,16 +135,15 @@ func Run(cfg *config.YurtHubConfiguration, stopCh <-chan struct{}) error {
}
trace++

klog.Infof("%d. new restConfig manager for %s mode", trace, cfg.CertMgrMode)
restConfigMgr, err := hubrest.NewRestConfigManager(cfg, certManager, healthChecker)
klog.Infof("%d. new restConfig manager", trace)
restConfigMgr, err := hubrest.NewRestConfigManager(certManager, healthChecker)
if err != nil {
return fmt.Errorf("could not new restConfig manager, %w", err)
}
trace++

klog.Infof("%d. create tls config for secure servers ", trace)
cfg.TLSConfig, err = server.GenUseCertMgrAndTLSConfig(
restConfigMgr, certManager, filepath.Join(cfg.RootDir, "pki"), cfg.NodeName, cfg.CertIPs, stopCh)
cfg.TLSConfig, err = server.GenUseCertMgrAndTLSConfig(certManager)
if err != nil {
return fmt.Errorf("could not create tls config, %w", err)
}
Expand Down
4 changes: 2 additions & 2 deletions pkg/controller/certificates/csrapprover.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ import (
"k8s.io/klog/v2"

"github.com/openyurtio/openyurt/pkg/projectinfo"
"github.com/openyurtio/openyurt/pkg/yurthub/certificate/hubself"
"github.com/openyurtio/openyurt/pkg/yurthub/certificate/token"
"github.com/openyurtio/openyurt/pkg/yurttunnel/constants"
)

Expand Down Expand Up @@ -432,7 +432,7 @@ func isYurtHubNodeCert(csr *certificatesv1.CertificateSigningRequest, x509cr *x5
return false
} else {
for _, org := range x509cr.Subject.Organization {
if org != hubself.YurtHubCSROrg && org != user.NodesGroup && !strings.HasPrefix(org, yurtHubNodeCertOrgPrefix) {
if org != token.YurtHubCSROrg && org != user.NodesGroup && !strings.HasPrefix(org, yurtHubNodeCertOrgPrefix) {
return false
}
}
Expand Down
6 changes: 3 additions & 3 deletions pkg/controller/certificates/csrapprover_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ import (
"k8s.io/client-go/util/cert"
"k8s.io/klog/v2"

"github.com/openyurtio/openyurt/pkg/yurthub/certificate/hubself"
"github.com/openyurtio/openyurt/pkg/yurthub/certificate/token"
"github.com/openyurtio/openyurt/pkg/yurttunnel/constants"
)

Expand Down Expand Up @@ -91,7 +91,7 @@ func TestIsYurtCSR(t *testing.T) {
certificatesv1.UsageKeyEncipherment,
certificatesv1.UsageClientAuth,
},
Request: newCSRData("system:node:xxx", []string{hubself.YurtHubCSROrg, user.NodesGroup, "openyurt:tenant:xxx"}, []string{}, []net.IP{}),
Request: newCSRData("system:node:xxx", []string{token.YurtHubCSROrg, user.NodesGroup, "openyurt:tenant:xxx"}, []string{}, []net.IP{}),
},
},
exp: true,
Expand All @@ -106,7 +106,7 @@ func TestIsYurtCSR(t *testing.T) {
certificatesv1.UsageKeyEncipherment,
certificatesv1.UsageClientAuth,
},
Request: newCSRData("system:node:xxx", []string{hubself.YurtHubCSROrg, user.NodesGroup, "unknown org"}, []string{}, []net.IP{}),
Request: newCSRData("system:node:xxx", []string{token.YurtHubCSROrg, user.NodesGroup, "unknown org"}, []string{}, []net.IP{}),
},
},
exp: false,
Expand Down
7 changes: 4 additions & 3 deletions pkg/node-servant/components/yurthub.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,11 @@ import (
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/klog/v2"

"github.com/openyurtio/openyurt/pkg/projectinfo"
"github.com/openyurtio/openyurt/pkg/util/templates"
constants "github.com/openyurtio/openyurt/pkg/yurtadm/constants"
"github.com/openyurtio/openyurt/pkg/yurtadm/constants"
enutil "github.com/openyurtio/openyurt/pkg/yurtadm/util/edgenode"
"github.com/openyurtio/openyurt/pkg/yurthub/certificate/hubself"
"github.com/openyurtio/openyurt/pkg/yurthub/certificate/token"
"github.com/openyurtio/openyurt/pkg/yurthub/storage/disk"
"github.com/openyurtio/openyurt/pkg/yurthub/util"
)
Expand Down Expand Up @@ -150,7 +151,7 @@ func getYurthubYaml(podManifestPath string) string {
}

func getYurthubConf() string {
return filepath.Join(hubself.HubRootDir, hubself.HubName)
return filepath.Join(token.DefaultRootDir, projectinfo.GetHubName())
}

func getYurthubCacheDir() string {
Expand Down
33 changes: 23 additions & 10 deletions pkg/util/certmanager/factory/factory.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"k8s.io/client-go/util/certificate"
"k8s.io/klog/v2"

"github.com/openyurtio/openyurt/pkg/util"
"github.com/openyurtio/openyurt/pkg/util/certmanager/store"
)

Expand Down Expand Up @@ -78,20 +79,33 @@ type CertManagerFactory interface {
New(*CertManagerConfig) (certificate.Manager, error)
}

type factory struct {
clientsetFn certificate.ClientsetFunc
fileStore certificate.FileStore
}

func NewCertManagerFactory(clientSet kubernetes.Interface) CertManagerFactory {
return &factory{
clientset: clientSet,
clientsetFn: func(current *tls.Certificate) (kubernetes.Interface, error) {
return clientSet, nil
},
}
}

type factory struct {
clientset kubernetes.Interface
func NewCertManagerFactoryWithFnAndStore(clientsetFn certificate.ClientsetFunc, store certificate.FileStore) CertManagerFactory {
return &factory{
clientsetFn: clientsetFn,
fileStore: store,
}
}

func (f *factory) New(cfg *CertManagerConfig) (certificate.Manager, error) {
store, err := store.NewFileStoreWrapper(cfg.ComponentName, cfg.CertDir, cfg.CertDir, "", "")
if err != nil {
return nil, fmt.Errorf("failed to initialize the server certificate store: %w", err)
var err error
if util.IsNil(f.fileStore) {
f.fileStore, err = store.NewFileStoreWrapper(cfg.ComponentName, cfg.CertDir, cfg.CertDir, "", "")
if err != nil {
return nil, fmt.Errorf("failed to initialize the server certificate store: %w", err)
}
}

ips, dnsNames := cfg.IPs, cfg.DNSNames
Expand Down Expand Up @@ -139,12 +153,11 @@ func (f *factory) New(cfg *CertManagerConfig) (certificate.Manager, error) {
}

return certificate.NewManager(&certificate.Config{
ClientsetFn: func(current *tls.Certificate) (kubernetes.Interface, error) {
return f.clientset, nil
},
ClientsetFn: f.clientsetFn,
SignerName: cfg.SignerName,
GetTemplate: getTemplate,
Usages: usages,
CertificateStore: store,
CertificateStore: f.fileStore,
Logf: klog.Infof,
})
}
Loading