feature(main): join nodes for k3s

Signed-off-by: cuisongliu <cuisongliu@qq.com>

pkg/runtime/k3s/bootstrap.go

@@ -21,6 +21,8 @@ import (
 
 	"golang.org/x/sync/errgroup"
 
+	"github.com/labring/sealos/pkg/utils/iputils"
+
 	"github.com/labring/sealos/pkg/constants"
 	"github.com/labring/sealos/pkg/utils/file"
 	"github.com/labring/sealos/pkg/utils/logger"
@@ -36,6 +38,9 @@ func (k *K3s) initMaster0() error {
 		k.generateAndSendInitConfig,
 		func() error { return k.enableK3sService(master0) },
 		k.pullKubeConfigFromMaster0,
+		func() error {
+			return k.remoteUtil.HostsAdd(master0, iputils.GetHostIP(master0), constants.DefaultAPIServerDomain)
+		},
 		func() error { return k.copyKubeConfigFileToNodes(k.cluster.GetMaster0IPAndPort()) },
 	)
 }
@@ -95,6 +100,9 @@ func (k *K3s) joinMaster(master string) error {
 			return k.execer.Copy(master, filepath.Join(k.pathResolver.EtcPath(), defaultJoinMastersFilename), defaultK3sConfigPath)
 		},
 		func() error { return k.enableK3sService(master) },
+		func() error {
+			return k.remoteUtil.HostsAdd(master, iputils.GetHostIP(master), constants.DefaultAPIServerDomain)
+		},
 		func() error { return k.copyKubeConfigFileToNodes(master) },
 	)
 }
@@ -111,8 +119,34 @@ func (k *K3s) joinNodes(nodes []string) error {
 	return nil
 }
 
+func (k *K3s) getAPIServerPort() int {
+	src := filepath.Join(k.pathResolver.EtcPath(), defaultInitFilename)
+	if file.IsExist(src) {
+		cfg := &Config{}
+		if err := yaml.UnmarshalFile(src, cfg); err == nil {
+			return cfg.HTTPSPort
+		}
+	}
+	return constants.DefaultAPIServerPort
+}
+
+func (k *K3s) getMasterIPListAndHTTPSPort() []string {
+	masters := make([]string, 0)
+	for _, master := range k.cluster.GetMasterIPList() {
+		masters = append(masters, fmt.Sprintf("%s:%d", master, k.getAPIServerPort()))
+	}
+	return masters
+}
+
+func (k *K3s) getVipAndPort() string {
+	return fmt.Sprintf("%s:%d", k.cluster.GetVIP(), k.getAPIServerPort())
+}
+
 func (k *K3s) joinNode(node string) error {
 	return k.runPipelines(fmt.Sprintf("join node %s", node),
+		func() error {
+			return k.remoteUtil.IPVS(node, k.getVipAndPort(), k.getMasterIPListAndHTTPSPort())
+		},
 		func() error { return k.generateAndSendTokenFiles(node, "agent-token") },
 		func() error {
 			return k.execer.Copy(node, filepath.Join(k.pathResolver.EtcPath(), defaultJoinNodesFilename), defaultK3sConfigPath)
@@ -193,6 +227,14 @@ func (k *K3s) pullKubeConfigFromMaster0() error {
 
 func (k *K3s) copyKubeConfigFileToNodes(hosts ...string) error {
 	src := k.pathResolver.AdminFile()
+	//data, err := file.ReadAll(src)
+	//if err != nil {
+	//	return errors.WithMessage(err, "read admin.config file failed")
+	//}
+	//newData := strings.ReplaceAll(string(data), "https://0.0.0.0:6443", fmt.Sprintf("https://%s:%d", constants.DefaultAPIServerDomain, 6443))
+	//if err = file.WriteFile(src, []byte(newData)); err != nil {
+	//	return errors.WithMessage(err, "write admin.config file failed")
+	//}
 	eg, _ := errgroup.WithContext(context.Background())
 	for _, node := range hosts {
 		node := node

pkg/runtime/k3s/config.go

@@ -37,9 +37,8 @@ var defaultMergeOpts = []func(*mergo.Config){
 }
 
 func defaultingConfig(c *Config) *Config {
-	//TODO update kube config
 	c.BindAddress = "0.0.0.0"
-	c.HTTPSPort = 6443
+	c.HTTPSPort = constants.DefaultAPIServerPort
 	c.ClusterCIDR = []string{"10.42.0.0/16"}
 	c.ServiceCIDR = []string{"10.96.0.0/16"}
 	c.ClusterDomain = constants.DefaultDNSDomain
@@ -173,16 +172,6 @@ func (k *K3s) getInitConfig(callbacks ...callback) (*Config, error) {
 	return cfg, nil
 }
 
-//lint:ignore U1000 Ignore unused function temporarily for debugging
-func (c *Config) getContainerRuntimeEndpoint() string {
-	if c.AgentConfig.Docker {
-		return "unix:///run/k3s/cri-dockerd/cri-dockerd.sock"
-	} else if len(c.AgentConfig.ContainerRuntimeEndpoint) == 0 {
-		return "unix:///run/k3s/containerd/containerd.sock"
-	}
-	return c.AgentConfig.ContainerRuntimeEndpoint
-}
-
 // ParseConfig return nil if data structure is not matched
 func ParseConfig(data []byte) (*Config, error) {
 	var cfg Config

pkg/runtime/k3s/consts.go

@@ -25,7 +25,7 @@ const (
 	defaultInitFilename        = "k3s-init.yaml"
 	defaultJoinMastersFilename = "k3s-join-master.yaml"
 	defaultJoinNodesFilename   = "k3s-join-node.yaml"
-	defaultPodManifestPath     = "pod-manifests"
+	k3sEtcStaticPod            = "/var/lib/rancher/k3s/agent/pod-manifests"
 )
 
 const (

pkg/runtime/k3s/k3s.go

@@ -15,8 +15,14 @@
 package k3s
 
 import (
+	"context"
 	"fmt"
 
+	"golang.org/x/sync/errgroup"
+
+	"github.com/labring/sealos/pkg/utils/iputils"
+	"github.com/labring/sealos/pkg/utils/strings"
+
 	"github.com/labring/sealos/pkg/constants"
 	"github.com/labring/sealos/pkg/env"
 	"github.com/labring/sealos/pkg/exec"
@@ -115,9 +121,26 @@ func (k *K3s) GetRawConfig() ([]byte, error) {
 	return yaml.MarshalConfigs(cluster, cfg)
 }
 
-func (k *K3s) SyncNodeIPVS(_, _ []string) error {
-	logger.Error("not yet implemented, skip for testing")
-	return nil
+func (k *K3s) SyncNodeIPVS(mastersIPList, nodeIPList []string) error {
+	mastersIPList = strings.RemoveDuplicate(mastersIPList)
+	masters := make([]string, 0)
+	for _, master := range mastersIPList {
+		masters = append(masters, fmt.Sprintf("%s:%d", iputils.GetHostIP(master), k.getAPIServerPort()))
+	}
+	image := k.cluster.GetLvscareImage()
+	eg, _ := errgroup.WithContext(context.Background())
+	for _, node := range nodeIPList {
+		node := node
+		eg.Go(func() error {
+			logger.Info("start to sync lvscare static pod to node: %s master: %+v", node, masters)
+			err := k.remoteUtil.StaticPod(node, k.getVipAndPort(), constants.LvsCareStaticPodName, image, masters, k3sEtcStaticPod)
+			if err != nil {
+				return fmt.Errorf("update lvscare static pod failed %s %v", node, err)
+			}
+			return nil
+		})
+	}
+	return eg.Wait()
 }
 
 func (k *K3s) runPipelines(phase string, pipelines ...func() error) error {

pkg/runtime/kubernetes/runtime_getter.go

@@ -112,7 +112,7 @@ func (k *KubeadmRuntime) syncNodeIPVSYaml(masterIPs, nodesIPs []string) error {
 
 func (k *KubeadmRuntime) execIPVSPod(ip string, masters []string) error {
 	image := k.cluster.GetLvscareImage()
-	return k.remoteUtil.StaticPod(ip, k.getVipAndPort(), constants.LvsCareStaticPodName, image, masters)
+	return k.remoteUtil.StaticPod(ip, k.getVipAndPort(), constants.LvsCareStaticPodName, image, masters, kubernetesEtcStaticPod)
 }
 
 func (k *KubeadmRuntime) execToken(ip, certificateKey string) (string, error) {

pkg/ssh/remote.go

@@ -86,13 +86,14 @@ func (s *Remote) IPVSClean(ip, vip string) error {
 	return s.executeRemoteUtilSubcommand(ip, out)
 }
 
-func (s *Remote) StaticPod(ip, vip, name, image string, masters []string) error {
-	staticPodIPVSTemplate := `static-pod lvscare --name {{.name}} --vip {{.vip}} --image {{.image}}  {{range $h := .masters}} --masters  {{$h}}{{end}}`
+func (s *Remote) StaticPod(ip, vip, name, image string, masters []string, path string) error {
+	staticPodIPVSTemplate := `static-pod lvscare --path {{.path}} --name {{.name}} --vip {{.vip}} --image {{.image}}  {{range $h := .masters}} --masters  {{$h}}{{end}}`
 	data := map[string]interface{}{
 		"vip":     vip,
 		"image":   image,
 		"masters": masters,
 		"name":    name,
+		"path":    path,
 	}
 	out, err := template.RenderTemplate("lvscare", staticPodIPVSTemplate, data)
 	if err != nil {