Merge pull request kosmos-io#637 from OrangeBao/hot-fix-release-0.4.0

feat: support ipv6

pkg/kubenest/constants/constant.go

@@ -1,25 +1,30 @@
 package constants
 
-import "time"
+import (
+	"time"
+
+	"github.com/kosmos.io/kosmos/pkg/utils"
+)
 
 const (
-	InitControllerName            = "virtual-cluster-init-controller"
-	NodeControllerName            = "virtual-cluster-node-controller"
-	GlobalNodeControllerName      = "global-node-controller"
-	KosmosJoinControllerName      = "kosmos-join-controller"
-	KosmosNs                      = "kosmos-system"
-	SystemNs                      = "kube-system"
-	DefauleImageRepositoryEnv     = "IMAGE_REPOSITIRY"
-	DefauleImageVersionEnv        = "IMAGE_VERSION"
-	VirtualClusterFinalizerName   = "kosmos.io/virtual-cluster-finalizer"
-	ServiceType                   = "NodePort"
-	EtcdServiceType               = "ClusterIP"
-	DisableCascadingDeletionLabel = "operator.virtualcluster.io/disable-cascading-deletion"
-	ControllerFinalizerName       = "operator.virtualcluster.io/finalizer"
-	DefaultKubeconfigPath         = "/etc/cluster-tree/cert"
-	Label                         = "virtualCluster-app"
-	ComponentBeReadyTimeout       = 300 * time.Second
-	ComponentBeDeletedTimeout     = 300 * time.Second
+	InitControllerName               = "virtual-cluster-init-controller"
+	NodeControllerName               = "virtual-cluster-node-controller"
+	GlobalNodeControllerName         = "global-node-controller"
+	KosmosJoinControllerName         = "kosmos-join-controller"
+	KosmosNs                         = "kosmos-system"
+	SystemNs                         = "kube-system"
+	DefauleImageRepositoryEnv        = "IMAGE_REPOSITIRY"
+	DefauleImageVersionEnv           = "IMAGE_VERSION"
+	DefauleVirtualControllerLabelEnv = "VIRTUAL_CONTROLLER_LABEL"
+	VirtualClusterFinalizerName      = "kosmos.io/virtual-cluster-finalizer"
+	ServiceType                      = "NodePort"
+	EtcdServiceType                  = "ClusterIP"
+	DisableCascadingDeletionLabel    = "operator.virtualcluster.io/disable-cascading-deletion"
+	ControllerFinalizerName          = "operator.virtualcluster.io/finalizer"
+	DefaultKubeconfigPath            = "/etc/cluster-tree/cert"
+	Label                            = "virtualCluster-app"
+	ComponentBeReadyTimeout          = 300 * time.Second
+	ComponentBeDeletedTimeout        = 300 * time.Second
 
 	// CertificateBlockType is a possible value for pem.Block.Type.
 	CertificateBlockType           = "CERTIFICATE"
@@ -41,7 +46,6 @@ const (
 	//controlplane apiserver
 	ApiServer                     = "apiserver"
 	ApiServerAnp                  = "apiserver-anp"
-	ApiServerServiceSubnet        = "10.237.6.0/18"
 	ApiServerEtcdListenClientPort = 2379
 	ApiServerServiceType          = "NodePort"
 	// APICallRetryInterval defines how long kubeadm should wait before retrying a failed API operation
@@ -114,3 +118,12 @@ const (
 )
 
 type Action string
+
+var ApiServerServiceSubnet string
+var KubeControllerManagerPodSubnet string
+
+func init() {
+	ApiServerServiceSubnet = utils.GetEnvWithDefaultValue("SERVICE_SUBNET", "10.237.6.0/18")
+	// fd11:1122:1111::/48,
+	KubeControllerManagerPodSubnet = utils.GetEnvWithDefaultValue("POD_SUBNET", "10.244.0.0/16")
+}

pkg/kubenest/controller/virtualcluster.node.controller/env/env.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"strconv"
+	"strings"
 
 	"k8s.io/klog"
 )
@@ -147,3 +148,11 @@ func GetNodeTaskMaxGoroutines() int {
 	}
 	return num
 }
+
+func GetCMDPaths() []string {
+	cmdAbsolutePaths := os.Getenv("CMD_ABSOLUTE_PATHS")
+	if len(cmdAbsolutePaths) == 0 {
+		return nil
+	}
+	return strings.Split(cmdAbsolutePaths, ",")
+}

pkg/kubenest/controller/virtualcluster.node.controller/exector/exector.go

@@ -11,6 +11,7 @@ import (
 	"k8s.io/klog/v2"
 
 	env "github.com/kosmos.io/kosmos/pkg/kubenest/controller/virtualcluster.node.controller/env"
+	"github.com/kosmos.io/kosmos/pkg/utils"
 )
 
 type Status int
@@ -175,6 +176,6 @@ func NewExectorHelper(addr string, port string) *ExectorHelper {
 	token := env.GetExectorToken()
 	return &ExectorHelper{
 		Token: token,
-		Addr:  fmt.Sprintf("%s:%s", addr, exectorPort),
+		Addr:  utils.GenerateAddrStr(addr, exectorPort),
 	}
 }

pkg/kubenest/controller/virtualcluster.node.controller/exector/remote_cmd.go

@@ -1,19 +1,35 @@
 package exector
 
 import (
+	"fmt"
 	"strings"
 
 	"github.com/gorilla/websocket"
+
+	env "github.com/kosmos.io/kosmos/pkg/kubenest/controller/virtualcluster.node.controller/env"
 )
 
 type CMDExector struct {
 	Cmd string
 }
 
+func AddPrefix(cmd string) string {
+	cmdAbsolutePaths := env.GetCMDPaths()
+	if len(cmdAbsolutePaths) == 0 {
+		return cmd
+	}
+	for _, cmdAbsolutePath := range cmdAbsolutePaths {
+		if strings.HasSuffix(cmdAbsolutePath, fmt.Sprintf("/%s", cmd)) {
+			return cmdAbsolutePath
+		}
+	}
+	return cmd
+}
+
 func (e *CMDExector) GetWebSocketOption() WebSocketOption {
 	cmdArgs := strings.Split(e.Cmd, " ")
 	command := cmdArgs[0]
-	rawQuery := "command=" + command
+	rawQuery := "command=" + AddPrefix(command)
 	if len(cmdArgs) > 1 {
 		args := cmdArgs[1:]
 		rawQuery = rawQuery + "&args=" + strings.Join(args, "&args=")

pkg/kubenest/controlplane/apiserver.go

@@ -36,18 +36,27 @@ func installAPIServer(client clientset.Interface, name, namespace string, portMa
 		return nil
 	}
 
+	vclabel := util.GetVirtualControllerLabel()
+
+	IPV6FirstFlag, err := util.IPV6First(constants.ApiServerServiceSubnet)
+	if err != nil {
+		return err
+	}
+
 	apiserverDeploymentBytes, err := util.ParseTemplate(apiserver.ApiserverDeployment, struct {
-		DeploymentName, Namespace, ImageRepository, EtcdClientService, Version string
-		ServiceSubnet, VirtualClusterCertsSecret, EtcdCertsSecret              string
-		Replicas                                                               int
-		EtcdListenClientPort                                                   int32
-		ClusterPort                                                            int32
-		AdmissionPlugins                                                       bool
+		DeploymentName, Namespace, ImageRepository, EtcdClientService, Version, VirtualControllerLabel string
+		ServiceSubnet, VirtualClusterCertsSecret, EtcdCertsSecret                                      string
+		Replicas                                                                                       int
+		EtcdListenClientPort                                                                           int32
+		ClusterPort                                                                                    int32
+		AdmissionPlugins                                                                               bool
+		IPV6First                                                                                      bool
 	}{
 		DeploymentName:            fmt.Sprintf("%s-%s", name, "apiserver"),
 		Namespace:                 namespace,
 		ImageRepository:           imageRepository,
 		Version:                   imageVersion,
+		VirtualControllerLabel:    vclabel,
 		EtcdClientService:         clusterIp,
 		ServiceSubnet:             constants.ApiServerServiceSubnet,
 		VirtualClusterCertsSecret: fmt.Sprintf("%s-%s", name, "cert"),
@@ -56,6 +65,7 @@ func installAPIServer(client clientset.Interface, name, namespace string, portMa
 		EtcdListenClientPort:      constants.ApiServerEtcdListenClientPort,
 		ClusterPort:               portMap[constants.ApiServerPortKey],
 		AdmissionPlugins:          opt.AdmissionPlugins,
+		IPV6First:                 IPV6FirstFlag,
 	})
 	if err != nil {
 		return fmt.Errorf("error when parsing virtual cluster apiserver deployment template: %w", err)

pkg/kubenest/controlplane/component.go

@@ -113,19 +113,31 @@ func getComponentConfigmaps(component string) []string {
 
 func getKubeControllerManagerManifest(name, namespace string) (*appsv1.Deployment, error) {
 	imageRepository, imageVersion := util.GetImageMessage()
+
+	vclabel := util.GetVirtualControllerLabel()
+
+	IPV6FirstFlag, err := util.IPV6First(constants.ApiServerServiceSubnet)
+	if err != nil {
+		return nil, err
+	}
+
 	kubeControllerManagerBytes, err := util.ParseTemplate(controller.KubeControllerManagerDeployment, struct {
-		DeploymentName, Namespace, ImageRepository, Version        string
-		VirtualClusterCertsSecret, KubeconfigSecret, ServiceSubnet string
-		Replicas                                                   int32
+		DeploymentName, Namespace, ImageRepository, Version, VirtualControllerLabel string
+		VirtualClusterCertsSecret, KubeconfigSecret, ServiceSubnet, PodSubnet       string
+		Replicas                                                                    int32
+		IPV6First                                                                   bool
 	}{
 		DeploymentName:            fmt.Sprintf("%s-%s", name, "kube-controller-manager"),
 		Namespace:                 namespace,
 		ImageRepository:           imageRepository,
 		Version:                   imageVersion,
+		VirtualControllerLabel:    vclabel,
 		VirtualClusterCertsSecret: fmt.Sprintf("%s-%s", name, "cert"),
 		KubeconfigSecret:          fmt.Sprintf("%s-%s", name, "admin-config-clusterip"),
 		ServiceSubnet:             constants.ApiServerServiceSubnet,
+		PodSubnet:                 constants.KubeControllerManagerPodSubnet,
 		Replicas:                  constants.KubeControllerReplicas,
+		IPV6First:                 IPV6FirstFlag,
 	})
 	if err != nil {
 		return nil, fmt.Errorf("error when parsing kube-controller-manager deployment template: %w", err)
@@ -160,18 +172,20 @@ func getVirtualClusterSchedulerConfigMapManifest(name, namespace string) (*v1.Co
 
 func getVirtualClusterSchedulerManifest(name, namespace string) (*appsv1.Deployment, error) {
 	imageRepository, imageVersion := util.GetImageMessage()
+	vclabel := util.GetVirtualControllerLabel()
 	virtualClusterSchedulerBytes, err := util.ParseTemplate(scheduler.VirtualClusterSchedulerDeployment, struct {
-		Replicas                                                             int32
-		DeploymentName, Namespace, SystemNamespace, ImageRepository, Version string
-		Image, KubeconfigSecret                                              string
+		Replicas                                                                                     int32
+		DeploymentName, Namespace, SystemNamespace, ImageRepository, Version, VirtualControllerLabel string
+		Image, KubeconfigSecret                                                                      string
 	}{
-		DeploymentName:   fmt.Sprintf("%s-%s", name, "virtualcluster-scheduler"),
-		Namespace:        namespace,
-		SystemNamespace:  constants.SystemNs,
-		ImageRepository:  imageRepository,
-		Version:          imageVersion,
-		KubeconfigSecret: fmt.Sprintf("%s-%s", name, "admin-config-clusterip"),
-		Replicas:         constants.VirtualClusterSchedulerReplicas,
+		DeploymentName:         fmt.Sprintf("%s-%s", name, "virtualcluster-scheduler"),
+		Namespace:              namespace,
+		SystemNamespace:        constants.SystemNs,
+		ImageRepository:        imageRepository,
+		VirtualControllerLabel: vclabel,
+		Version:                imageVersion,
+		KubeconfigSecret:       fmt.Sprintf("%s-%s", name, "admin-config-clusterip"),
+		Replicas:               constants.VirtualClusterSchedulerReplicas,
 	})
 	if err != nil {
 		return nil, fmt.Errorf("error when parsing virtualCluster-scheduler deployment template: %w", err)

pkg/kubenest/controlplane/etcd.go

@@ -46,25 +46,35 @@ func installEtcd(client clientset.Interface, name, namespace string) error {
 		initialClusters[index] = fmt.Sprintf("%s=%s", memberName, memberPeerURL)
 	}
 
+	vclabel := util.GetVirtualControllerLabel()
+
+	IPV6FirstFlag, err := util.IPV6First(constants.ApiServerServiceSubnet)
+	if err != nil {
+		return err
+	}
+
 	etcdStatefulSetBytes, err := util.ParseTemplate(etcd.EtcdStatefulSet, struct {
-		StatefulSetName, Namespace, ImageRepository, Image, EtcdClientService, Version string
-		CertsSecretName, EtcdPeerServiceName                                           string
-		InitialCluster, EtcdDataVolumeName, EtcdCipherSuites                           string
-		Replicas, EtcdListenClientPort, EtcdListenPeerPort                             int32
+		StatefulSetName, Namespace, ImageRepository, Image, EtcdClientService, Version, VirtualControllerLabel string
+		CertsSecretName, EtcdPeerServiceName                                                                   string
+		InitialCluster, EtcdDataVolumeName, EtcdCipherSuites                                                   string
+		Replicas, EtcdListenClientPort, EtcdListenPeerPort                                                     int32
+		IPV6First                                                                                              bool
 	}{
-		StatefulSetName:      fmt.Sprintf("%s-%s", name, "etcd"),
-		Namespace:            namespace,
-		ImageRepository:      imageRepository,
-		Version:              imageVersion,
-		EtcdClientService:    fmt.Sprintf("%s-%s", name, "etcd-client"),
-		CertsSecretName:      fmt.Sprintf("%s-%s", name, "etcd-cert"),
-		EtcdPeerServiceName:  fmt.Sprintf("%s-%s", name, "etcd"),
-		EtcdDataVolumeName:   constants.EtcdDataVolumeName,
-		InitialCluster:       strings.Join(initialClusters, ","),
-		EtcdCipherSuites:     strings.Join(flag.PreferredTLSCipherNames(), ","),
-		Replicas:             constants.EtcdReplicas,
-		EtcdListenClientPort: constants.EtcdListenClientPort,
-		EtcdListenPeerPort:   constants.EtcdListenPeerPort,
+		StatefulSetName:        fmt.Sprintf("%s-%s", name, "etcd"),
+		Namespace:              namespace,
+		ImageRepository:        imageRepository,
+		Version:                imageVersion,
+		VirtualControllerLabel: vclabel,
+		EtcdClientService:      fmt.Sprintf("%s-%s", name, "etcd-client"),
+		CertsSecretName:        fmt.Sprintf("%s-%s", name, "etcd-cert"),
+		EtcdPeerServiceName:    fmt.Sprintf("%s-%s", name, "etcd"),
+		EtcdDataVolumeName:     constants.EtcdDataVolumeName,
+		InitialCluster:         strings.Join(initialClusters, ","),
+		EtcdCipherSuites:       strings.Join(flag.PreferredTLSCipherNames(), ","),
+		Replicas:               constants.EtcdReplicas,
+		EtcdListenClientPort:   constants.EtcdListenClientPort,
+		EtcdListenPeerPort:     constants.EtcdListenPeerPort,
+		IPV6First:              IPV6FirstFlag,
 	})
 	if err != nil {
 		return fmt.Errorf("error when parsing Etcd statefuelset template: %w", err)

pkg/kubenest/manifest/controlplane/apiserver/mainfests_deployment.go

@@ -24,16 +24,16 @@ spec:
       hostNetwork: true
       dnsPolicy: ClusterFirstWithHostNet
       tolerations:
-      - key: "node-role.kubernetes.io/control-plane"
+      - key: {{ .VirtualControllerLabel }}
         operator: "Exists"
         effect: "NoSchedule"
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
               - matchExpressions:
-                  - key: node-role.kubernetes.io/control-plane
-                    operator: Exists
+                - key: {{ .VirtualControllerLabel }}
+                  operator: Exists
         podAntiAffinity:
           preferredDuringSchedulingIgnoredDuringExecution:
           - weight: 100
@@ -66,8 +66,12 @@ spec:
         - --etcd-certfile=/etc/etcd/pki/etcd-client.crt
         - --etcd-keyfile=/etc/etcd/pki/etcd-client.key
         #- --etcd-servers=https://{{ .EtcdClientService }}.{{ .Namespace }}.svc.cluster.local:{{ .EtcdListenClientPort }}
+        {{ if .IPV6First }}
+        - --etcd-servers=https://[{{ .EtcdClientService }}]:{{ .EtcdListenClientPort }}
+        {{ else }}
         - --etcd-servers=https://{{ .EtcdClientService }}:{{ .EtcdListenClientPort }}
-        - --bind-address=0.0.0.0
+        {{ end }}
+        - '--bind-address=::'
         - --kubelet-client-certificate=/etc/virtualcluster/pki/virtualCluster.crt
         - --kubelet-client-key=/etc/virtualcluster/pki/virtualCluster.key
         - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
@@ -160,16 +164,16 @@ spec:
       hostNetwork: true
       dnsPolicy: ClusterFirstWithHostNet
       tolerations:
-      - key: "node-role.kubernetes.io/control-plane"
+      - key: {{ .VirtualControllerLabel }}
         operator: "Exists"
         effect: "NoSchedule"
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
               - matchExpressions:
-                  - key: node-role.kubernetes.io/control-plane
-                    operator: Exists
+                - key: {{ .VirtualControllerLabel }}
+                  operator: Exists
         podAntiAffinity:
           preferredDuringSchedulingIgnoredDuringExecution:
           - weight: 100
@@ -202,8 +206,12 @@ spec:
         - --etcd-certfile=/etc/etcd/pki/etcd-client.crt
         - --etcd-keyfile=/etc/etcd/pki/etcd-client.key
         #- --etcd-servers=https://{{ .EtcdClientService }}.{{ .Namespace }}.svc.cluster.local:{{ .EtcdListenClientPort }}
+        {{ if .IPV6First }}
+        - --etcd-servers=https://[{{ .EtcdClientService }}]:{{ .EtcdListenClientPort }}
+        {{ else }}
         - --etcd-servers=https://{{ .EtcdClientService }}:{{ .EtcdListenClientPort }}
-        - --bind-address=0.0.0.0
+        {{ end }}
+        - '--bind-address=::'
         - --kubelet-client-certificate=/etc/virtualcluster/pki/virtualCluster.crt
         - --kubelet-client-key=/etc/virtualcluster/pki/virtualCluster.key
         - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname