feat: add anp install workflow

Signed-off-by: wangdepeng <wangdepeng_yewu@cmss.chinamobile.com>

deploy/virtual-cluster-operator.yml

@@ -533,5 +533,26 @@ data:
       - 33008
       - 33009
       - 33010
-    # when port is allocate from pool,it will be used for virtual cluster api server,and the port will be released after virtual cluster is deleted
-    clusterPorts:
+---
+apiVersion: v1
+data:
+  egress_selector_configuration.yaml: |
+    apiVersion: apiserver.k8s.io/v1beta1
+    kind: EgressSelectorConfiguration
+    egressSelections:
+      - name: cluster
+        connection:
+          proxyProtocol: GRPC
+          transport:
+            uds:
+              udsName: /etc/kubernetes/konnectivity-server/konnectivity-server.socket
+      - name: master
+        connection:
+          proxyProtocol: Direct
+      - name: etcd
+        connection:
+          proxyProtocol: Direct
+kind: ConfigMap
+metadata:
+  name: kas-proxy-files
+  namespace: kas-proxy

go.mod

@@ -48,6 +48,7 @@ require (
 	k8s.io/utils v0.0.0-20221128185143-99ec85e7a448
 	sigs.k8s.io/controller-runtime v0.14.5
 	sigs.k8s.io/mcs-api v0.1.0
+	sigs.k8s.io/yaml v1.3.0
 )
 
 require (
@@ -188,7 +189,6 @@ require (
 	sigs.k8s.io/kustomize/api v0.12.1 // indirect
 	sigs.k8s.io/kustomize/kyaml v0.13.9 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
-	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
 replace (

hack/generate_globalnode.sh

@@ -34,7 +34,7 @@ $(echo "${labelsFormatted}" | awk '{print "    " $0}')
     # Apply the YAML
     echo "$yamlContent" | kubectl apply -f -
 
-    # clear resources
-    rm -rf kube_apply_logs
 
 done
+# clear resources
+rm -rf kube_apply_logs

pkg/kubenest/constants/constant.go

@@ -39,7 +39,7 @@ const (
 
 	//controlplane apiserver
 	ApiServer                     = "apiserver"
-	ApiServerReplicas             = 2
+	ApiServerReplicas             = 1
 	ApiServerServiceSubnet        = "10.237.6.0/18"
 	ApiServerEtcdListenClientPort = 2379
 	ApiServerServiceType          = "NodePort"
@@ -84,11 +84,14 @@ const (
 	DeInitAction Action = "deInit"
 
 	//host_port_manager
-	HostPortsCMName              = "kosmos-hostports"
-	HostPortsCMDataName          = "config.yaml"
-	ApiServerPortKey             = "apiserver-port"
-	ApiServerNetworkProxyPortKey = "apiserver-network-proxy-port"
-	VirtualClusterPortNum        = 2
+	HostPortsCMName                    = "kosmos-hostports"
+	HostPortsCMDataName                = "config.yaml"
+	ApiServerPortKey                   = "apiserver-port"
+	ApiServerNetworkProxyAgentPortKey  = "apiserver-network-proxy-agent-port"
+	ApiServerNetworkProxyServerPortKey = "apiserver-network-proxy-server-port"
+	ApiServerNetworkProxyHealthPortKey = "apiserver-network-proxy-health-port"
+	ApiServerNetworkProxyAdminPortKey  = "apiserver-network-proxy-admin-port"
+	VirtualClusterPortNum              = 5
 
 	ManifestComponentsConfigMap = "components-manifest-cm"
 	NodePoolConfigmap           = "node-pool"

pkg/kubenest/controller/virtualcluster_init_controller.go

@@ -508,10 +508,14 @@ func (c *VirtualClusterInitController) isPortAllocated(port int32) bool {
 	return false
 }
 
+// AllocateHostPort allocate host port for virtual cluster
+// #nosec G602
 func (c *VirtualClusterInitController) AllocateHostPort(virtualCluster *v1alpha1.VirtualCluster) (int32, error) {
 	c.lock.Lock()
 	defer c.lock.Unlock()
-
+	if len(virtualCluster.Status.PortMap) > 0 || virtualCluster.Status.Port != 0 {
+		return 0, nil
+	}
 	hostPool, err := GetHostPortPoolFromConfigMap(c.RootClientSet, constants.KosmosNs, constants.HostPortsCMName, constants.HostPortsCMDataName)
 	if err != nil {
 		return 0, err
@@ -528,7 +532,14 @@ func (c *VirtualClusterInitController) AllocateHostPort(virtualCluster *v1alpha1
 	if len(ports) < constants.VirtualClusterPortNum {
 		return 0, fmt.Errorf("no available ports to allocate")
 	}
+	virtualCluster.Status.PortMap = make(map[string]int32)
 	virtualCluster.Status.PortMap[constants.ApiServerPortKey] = ports[0]
-	virtualCluster.Status.PortMap[constants.ApiServerNetworkProxyPortKey] = ports[1]
+	virtualCluster.Status.PortMap[constants.ApiServerNetworkProxyAgentPortKey] = ports[1]
+	virtualCluster.Status.PortMap[constants.ApiServerNetworkProxyServerPortKey] = ports[2]
+	virtualCluster.Status.PortMap[constants.ApiServerNetworkProxyHealthPortKey] = ports[3]
+	virtualCluster.Status.PortMap[constants.ApiServerNetworkProxyAdminPortKey] = ports[4]
+
+	klog.V(4).InfoS("Success allocate virtual cluster ports", "allocate ports", ports, "vc ports", ports[:2])
+
 	return 0, err
 }

pkg/kubenest/controlplane/apiserver.go

@@ -13,8 +13,8 @@ import (
 	"github.com/kosmos.io/kosmos/pkg/kubenest/util"
 )
 
-func EnsureVirtualClusterAPIServer(client clientset.Interface, name, namespace string, port int32) error {
-	if err := installAPIServer(client, name, namespace, port); err != nil {
+func EnsureVirtualClusterAPIServer(client clientset.Interface, name, namespace string, portMap map[string]int32) error {
+	if err := installAPIServer(client, name, namespace, portMap); err != nil {
 		return fmt.Errorf("failed to install virtual cluster apiserver, err: %w", err)
 	}
 	return nil
@@ -28,7 +28,7 @@ func DeleteVirtualClusterAPIServer(client clientset.Interface, name, namespace s
 	return nil
 }
 
-func installAPIServer(client clientset.Interface, name, namespace string, port int32) error {
+func installAPIServer(client clientset.Interface, name, namespace string, portMap map[string]int32) error {
 	imageRepository, imageVersion := util.GetImageMessage()
 	clusterIp, err := util.GetEtcdServiceClusterIp(namespace, name+constants.EtcdSuffix, client)
 	if err != nil {
@@ -52,7 +52,7 @@ func installAPIServer(client clientset.Interface, name, namespace string, port i
 		EtcdCertsSecret:           fmt.Sprintf("%s-%s", name, "etcd-cert"),
 		Replicas:                  constants.ApiServerReplicas,
 		EtcdListenClientPort:      constants.ApiServerEtcdListenClientPort,
-		ClusterPort:               port,
+		ClusterPort:               portMap[constants.ApiServerPortKey],
 	})
 	if err != nil {
 		return fmt.Errorf("error when parsing virtual cluster apiserver deployment template: %w", err)

pkg/kubenest/init.go

@@ -57,6 +57,7 @@ func NewInitPhase(opts *InitOptions) *workflow.Phase {
 	initPhase.AppendTask(tasks.NewVirtualClusterApiserverTask())
 	initPhase.AppendTask(tasks.NewUploadKubeconfigTask())
 	initPhase.AppendTask(tasks.NewCheckApiserverHealthTask())
+	initPhase.AppendTask(tasks.NewAnpTask())
 	initPhase.AppendTask(tasks.NewComponentTask())
 	initPhase.AppendTask(tasks.NewCheckControlPlaneTask())
 	// create core-dns
@@ -75,6 +76,7 @@ func UninstallPhase(opts *InitOptions) *workflow.Phase {
 	destroyPhase.AppendTask(tasks.UninstallCoreDNSTask())
 	destroyPhase.AppendTask(tasks.UninstallComponentTask())
 	destroyPhase.AppendTask(tasks.UninstallVirtualClusterApiserverTask())
+	destroyPhase.AppendTask(tasks.UninstallAnpTask())
 	destroyPhase.AppendTask(tasks.UninstallEtcdTask())
 	destroyPhase.AppendTask(tasks.UninstallVirtualClusterServiceTask())
 	destroyPhase.AppendTask(tasks.UninstallCertsAndKubeconfigTask())
@@ -171,6 +173,7 @@ func newRunData(opt *InitOptions) (*initData, error) {
 		CertStore:             cert.NewCertStore(),
 		externalIP:            opt.virtualCluster.Spec.ExternalIP,
 		hostPort:              opt.virtualCluster.Status.Port,
+		hostPortMap:           opt.virtualCluster.Status.PortMap,
 	}, nil
 }