update certificate automatically

cmd/yurt-tunnel-server/app/start.go

@@ -103,7 +103,7 @@ func Run(cfg *config.CompletedConfig, stopCh <-chan struct{}) error {
 
 	// 2. create a certificate manager for the tunnel server and run the
 	// csr approver for both yurttunnel-server and yurttunnel-agent
-	serverCertMgr, err := certmanager.NewYurttunnelServerCertManager(cfg.Client, cfg.CertDNSNames, cfg.CertIPs, stopCh)
+	serverCertMgr, err := certmanager.NewYurttunnelServerCertManager(cfg.Client, cfg.SharedInformerFactory, cfg.CertDNSNames, cfg.CertIPs, stopCh)
 	if err != nil {
 		return err
 	}

pkg/projectinfo/projectinfo.go

@@ -55,6 +55,11 @@ func GetServerName() string {
 	return projectPrefix + "tunnel-server"
 }
 
+// tunnel server label: yurt-tunnel-server
+func YurtTunnelServerLabel() string {
+	return projectPrefix + "-tunnel-server"
+}
+
 // Agent name: yurttunnel-agent
 func GetAgentName() string {
 	return projectPrefix + "tunnel-agent"

pkg/yurttunnel/dns/dns.go

@@ -137,9 +137,11 @@ func NewCoreDNSRecordController(client clientset.Interface,
 
 // newServiceInformer creates a shared index informer that returns only interested services
 func newServiceInformer(cs clientset.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
-	selector := fmt.Sprintf("metadata.name=%v", constants.YurttunnelServerInternalServiceName)
+	// this informer will be used by coreDNSRecordController and certificate manager,
+	// so it should return x-tunnel-server-svc and x-tunnel-server-internal-svc
+	selector := fmt.Sprintf("name=%v", projectinfo.YurtTunnelServerLabel())
 	tweakListOptions := func(options *metav1.ListOptions) {
-		options.FieldSelector = selector
+		options.LabelSelector = selector
 	}
 	return coreinformers.NewFilteredServiceInformer(cs, constants.YurttunnelServerServiceNs, resyncPeriod, nil, tweakListOptions)
 }

pkg/yurttunnel/pki/certmanager/certmanager.go

@@ -32,28 +32,37 @@ import (
 	"github.com/openyurtio/openyurt/pkg/yurttunnel/server/serveraddr"
 
 	certificates "k8s.io/api/certificates/v1beta1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/informers"
+	coreinformers "k8s.io/client-go/informers/core/v1"
 	"k8s.io/client-go/kubernetes"
 	clicert "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
+	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/util/certificate"
-	"k8s.io/klog/v2"
+	"k8s.io/klog"
 )
 
 // NewYurttunnelServerCertManager creates a certificate manager for
 // the yurttunnel-server
 func NewYurttunnelServerCertManager(
 	clientset kubernetes.Interface,
+	factory informers.SharedInformerFactory,
 	clCertNames []string,
 	clIPs []net.IP,
 	stopCh <-chan struct{}) (certificate.Manager, error) {
-	// get server DNS names and IPs
 	var (
 		dnsNames = []string{}
 		ips      = []net.IP{}
 		err      error
 	)
+
+	// add endPoints informer
+	factory.InformerFor(&v1.Endpoints{}, newEndPointsInformer)
+
+	// the ips and dnsNames should be acquired through api-server at the first time, because the informer factory has not started yet.
 	_ = wait.PollUntil(5*time.Second, func() (bool, error) {
 		dnsNames, ips, err = serveraddr.GetYurttunelServerDNSandIP(clientset)
 		if err != nil {
@@ -76,11 +85,18 @@ func NewYurttunnelServerCertManager(
 
 		return true, nil
 	}, stopCh)
-	// add user specified DNS anems and IP addresses
+	// add user specified DNS names and IP addresses
 	dnsNames = append(dnsNames, clCertNames...)
 	ips = append(ips, clIPs...)
 	klog.Infof("subject of tunnel server certificate, ips=%#+v, dnsNames=%#+v", ips, dnsNames)
 
+	// the dynamic ip acquire func
+	getIPs := func() ([]net.IP, error) {
+		_, dynamicIPs, err := serveraddr.YurttunnelServerAddrManager(factory)
+		dynamicIPs = append(dynamicIPs, clIPs...)
+		return dynamicIPs, err
+	}
+
 	return newCertManager(
 		clientset,
 		projectinfo.GetServerName(),
@@ -94,7 +110,8 @@ func NewYurttunnelServerCertManager(
 			certificates.UsageServerAuth,
 			certificates.UsageClientAuth,
 		},
-		ips)
+		ips,
+		getIPs)
 }
 
 // NewYurttunnelAgentCertManager creates a certificate manager for
@@ -121,7 +138,8 @@ func NewYurttunnelAgentCertManager(
 			certificates.UsageDigitalSignature,
 			certificates.UsageClientAuth,
 		},
-		[]net.IP{net.ParseIP(nodeIP)})
+		[]net.IP{net.ParseIP(nodeIP)},
+		nil)
 }
 
 // NewCertManager creates a certificate manager that will generates a
@@ -134,21 +152,29 @@ func newCertManager(
 	organizations,
 	dnsNames []string,
 	keyUsages []certificates.KeyUsage,
-	ipAddrs []net.IP) (certificate.Manager, error) {
+	ips []net.IP,
+	getIPs serveraddr.GetIPs) (certificate.Manager, error) {
 	certificateStore, err :=
 		store.NewFileStoreWrapper(componentName, certDir, certDir, "", "")
 	if err != nil {
 		return nil, fmt.Errorf("failed to initialize the server certificate store: %v", err)
 	}
 
 	getTemplate := func() *x509.CertificateRequest {
+		// use dynamic ips
+		if getIPs != nil {
+			tmpIPs, err := getIPs()
+			if err == nil {
+				ips = tmpIPs
+			}
+		}
 		return &x509.CertificateRequest{
 			Subject: pkix.Name{
 				CommonName:   commonName,
 				Organization: organizations,
 			},
 			DNSNames:    dnsNames,
-			IPAddresses: ipAddrs,
+			IPAddresses: ips,
 		}
 	}
 
@@ -167,3 +193,12 @@ func newCertManager(
 
 	return certManager, nil
 }
+
+// newEndPointsInformer creates a shared index informer that returns only interested endpoints
+func newEndPointsInformer(cs kubernetes.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
+	selector := fmt.Sprintf("metadata.name=%v", constants.YurttunnelEndpointsName)
+	tweakListOptions := func(options *metav1.ListOptions) {
+		options.FieldSelector = selector
+	}
+	return coreinformers.NewFilteredEndpointsInformer(cs, constants.YurttunnelEndpointsNs, resyncPeriod, nil, tweakListOptions)
+}

pkg/yurttunnel/server/serveraddr/addr.go

@@ -28,10 +28,14 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/klog/v2"
 )
 
+type GetIPs func() ([]net.IP, error)
+
 // GetServerAddr gets the service address that exposes the tunnel server for
 // tunnel agent to connect
 func GetTunnelServerAddr(clientset kubernetes.Interface) (string, error) {
@@ -47,7 +51,7 @@ func GetTunnelServerAddr(clientset kubernetes.Interface) (string, error) {
 		return "", err
 	}
 
-	dnsNames, ips, err := extractTunnelServerDNSandIPs(svc, eps, nodeLst)
+	dnsNames, ips, err := extractTunnelServerDNSandIPs(svc, eps, NodeListToNodes(nodeLst))
 	if err != nil {
 		return "", err
 	}
@@ -102,7 +106,62 @@ func GetYurttunelServerDNSandIP(
 		return []string{}, []net.IP{}, err
 	}
 
-	return extractTunnelServerDNSandIPs(svc, eps, nodeLst)
+	return extractTunnelServerDNSandIPs(svc, eps, NodeListToNodes(nodeLst))
+}
+
+// YurttunelServerAddrManager list the latest tunnel server resources, extract ips and dnsNames from them
+func YurttunnelServerAddrManager(factory informers.SharedInformerFactory) ([]string, []net.IP, error) {
+	var (
+		ips              = make([]net.IP, 0)
+		dnsNames         = make([]string, 0)
+		internalIp       net.IP
+		internalDnsNames = make([]string, 0)
+		err              error
+	)
+
+	// list yurt-tunnel-server services
+	services, err := factory.Core().V1().Services().Lister().List(labels.Everything())
+	if err != nil {
+		return dnsNames, ips, err
+	}
+
+	// list x-tunnel-server-svc endpoints
+	eps, err := factory.Core().V1().Endpoints().Lister().List(labels.Everything())
+	if err != nil {
+		return dnsNames, ips, err
+	}
+
+	// list all of cloud nodes
+	label := fmt.Sprintf("%s=false", projectinfo.GetEdgeWorkerLabelKey())
+	selector, _ := labels.Parse(label)
+	// yurttunnel-server will be deployed on one of the cloud nodes
+	nodes, err := factory.Core().V1().Nodes().Lister().List(selector)
+	if err != nil {
+		return dnsNames, ips, err
+	}
+
+	// extract ip from the services
+	for _, svc := range services {
+		if svc.Name == constants.YurttunnelServerServiceName {
+			dnsNames, ips, err = extractTunnelServerDNSandIPs(svc, eps[0], nodes)
+		} else {
+			// get clusterIP for x-tunnel-server-internal-svc
+			if svc.Name == constants.YurttunnelServerInternalServiceName && svc.Spec.ClusterIP != "" && net.ParseIP(svc.Spec.ClusterIP) != nil {
+				internalIp = net.ParseIP(svc.Spec.ClusterIP)
+				internalDnsNames = GetDefaultDomainsForSvc(svc.Namespace, svc.Name)
+			}
+		}
+	}
+
+	if internalIp != nil {
+		ips = append(ips, internalIp)
+	}
+
+	if len(internalDnsNames) != 0 {
+		dnsNames = append(dnsNames, internalDnsNames...)
+	}
+
+	return dnsNames, ips, err
 }
 
 // getTunnelServerResources get service, endpoints, and cloud nodes of tunnel server
@@ -143,14 +202,14 @@ func getTunnelServerResources(clientset kubernetes.Interface) (*v1.Service, *v1.
 }
 
 // extractTunnelServerDNSandIPs extract tunnel server dnses and ips from service and endpoints
-func extractTunnelServerDNSandIPs(svc *v1.Service, eps *v1.Endpoints, nodeLst *v1.NodeList) ([]string, []net.IP, error) {
+func extractTunnelServerDNSandIPs(svc *v1.Service, eps *v1.Endpoints, nodes []*v1.Node) ([]string, []net.IP, error) {
 	var (
 		dnsNames = make([]string, 0)
 		ips      = make([]net.IP, 0)
 		err      error
 	)
 
-	// extract dns and ip from the service
+	// 1. extract dns and ip from the service
 	switch svc.Spec.Type {
 	case corev1.ServiceTypeLoadBalancer:
 		// make sure lb ip address is the first index in return ips slice
@@ -159,7 +218,7 @@ func extractTunnelServerDNSandIPs(svc *v1.Service, eps *v1.Endpoints, nodeLst *v
 		// make sure annotation setting address is the first index in return ips slice
 		dnsNames, ips, err = getClusterIPDNSandIP(svc)
 	case corev1.ServiceTypeNodePort:
-		dnsNames, ips, err = getNodePortDNSandIP(nodeLst)
+		dnsNames, ips, err = getNodePortDNSandIP(nodes)
 	default:
 		err = fmt.Errorf("unsupported service type: %s", string(svc.Spec.Type))
 	}
@@ -168,14 +227,14 @@ func extractTunnelServerDNSandIPs(svc *v1.Service, eps *v1.Endpoints, nodeLst *v
 		return dnsNames, ips, err
 	}
 
-	// extract dns and ip from ClusterIP info
+	// 2. extract dns and ip from ClusterIP info
 	dnsNames = append(dnsNames, GetDefaultDomainsForSvc(svc.Namespace, svc.Name)...)
 	if svc.Spec.ClusterIP != "None" {
 		ips = append(ips, net.ParseIP(svc.Spec.ClusterIP))
 	}
 	ips = append(ips, net.ParseIP("127.0.0.1"))
 
-	// extract dns and ip from the endpoint
+	// 3. extract dns and ip from the endpoint
 	for _, ss := range eps.Subsets {
 		for _, addr := range ss.Addresses {
 			if len(addr.IP) != 0 {
@@ -241,27 +300,34 @@ func getClusterIPDNSandIP(svc *corev1.Service) ([]string, []net.IP, error) {
 }
 
 // getClusterIPDNSandIP gets the DNS names and IPs from the NodePort service
-func getNodePortDNSandIP(nodeLst *v1.NodeList) ([]string, []net.IP, error) {
+func getNodePortDNSandIP(nodes []*v1.Node) ([]string, []net.IP, error) {
 	var (
 		dnsNames = make([]string, 0)
 		ips      = make([]net.IP, 0)
 		ipFound  bool
 	)
 
-	if nodeLst == nil || len(nodeLst.Items) == 0 {
+	if len(nodes) == 0 {
 		return dnsNames, ips, errors.New("there is no cloud node")
 	}
 
-	for _, addr := range nodeLst.Items[0].Status.Addresses {
-		if addr.Type == corev1.NodeInternalIP {
-			ipFound = true
-			ips = append(ips, net.ParseIP(addr.Address))
+	for _, node := range nodes {
+		for _, addr := range node.Status.Addresses {
+			if addr.Type == corev1.NodeInternalIP {
+				ipFound = true
+				ips = append(ips, net.ParseIP(addr.Address))
+			}
+			if addr.Type == corev1.NodeHostName {
+				dnsNames = append(dnsNames, addr.Address)
+			}
 		}
 	}
+
 	if !ipFound {
 		// there is no qualified address (i.e. NodeInternalIP)
 		return dnsNames, ips, errors.New("can't find node IP")
 	}
+
 	return dnsNames, ips, nil
 }
 
@@ -279,3 +345,11 @@ func GetDefaultDomainsForSvc(ns, name string) []string {
 
 	return domains
 }
+
+func NodeListToNodes(nodeLst *v1.NodeList) []*v1.Node {
+	nodes := make([]*v1.Node, 0)
+	for _, node := range nodeLst.Items {
+		nodes = append(nodes, &node)
+	}
+	return nodes
+}