Merge pull request kosmos-io#651 from qiuwei68/release-0.4.0

cherry-pick: Add nodes back to the host cluster without kubeadm

hack/k8s-in-k8s/generate_env.sh

@@ -123,6 +123,10 @@ PATH_KUBELET_CONF=$PATH_KUBELET_CONF
 # name for config file of kubelet
 KUBELET_CONFIG_NAME=$KUBELET_CONFIG_NAME
 HOST_CORE_DNS=$HOST_CORE_DNS
+# kubeadm switch
+USE_KUBEADM=true
+# Generate kubelet.conf TIMEOUT
+KUBELET_CONF_TIMEOUT=30
 
 function GenerateKubeadmConfig() {
     echo \"---

hack/k8s-in-k8s/kubelet_node_helper.sh

@@ -89,6 +89,65 @@ function afterRevert() {
     fi
 }
 
+function get_ca_certificate() {
+     local output_file="$PATH_KUBERNETES_PKI/ca.crt"
+     local kubeconfig_data=$(curl -sS --insecure "https://$JOIN_HOST/api/v1/namespaces/kube-public/configmaps/cluster-info" 2>/dev/null | \
+                             grep -oP 'certificate-authority-data:\s*\K.*(?=server:[^[:space:]]*?)' | \
+                             sed -e 's/^certificate-authority-data://' -e 's/[[:space:]]//g' -e 's/\\n$//g')
+
+     # verify the kubeconfig data is not empty
+     if [ -z "$kubeconfig_data" ]; then
+       echo "Failed to extract certificate-authority-data."
+       return 1
+     fi
+
+     # Base64 decoded and written to a file
+     echo "$kubeconfig_data" | base64 --decode > "$output_file"
+
+     # check that the file was created successfully
+     if [ -f "$output_file" ]; then
+         echo "certificate-authority-data saved to $output_file"
+     else
+         echo "Failed to save certificate-authority-data to $output_file"
+      return 1
+     fi
+}
+
+function create_kubelet_bootstrap_config() {
+   # Checks if the parameters are provided
+ if [ -z "$JOIN_HOST" ] || [ -z "$JOIN_TOKEN" ]; then
+     echo "Please provide server and token as parameters."
+     return 1
+ fi
+
+ # Define file contents
+ cat << EOF > bootstrap-kubelet.conf
+apiVersion: v1
+kind: Config
+clusters:
+- cluster:
+    certificate-authority: $PATH_KUBERNETES_PKI/ca.crt
+    server: https://$JOIN_HOST
+  name: kubernetes
+contexts:
+- context:
+    cluster: kubernetes
+    user: kubelet-bootstrap
+  name: kubelet-bootstrap-context
+current-context: kubelet-bootstrap-context
+preferences: {}
+users:
+- name: kubelet-bootstrap
+  user:
+    token: $JOIN_TOKEN
+EOF
+
+ # copy the file to the /etc/kubernetes directory
+ cp bootstrap-kubelet.conf $PATH_KUBERNETES
+
+ echo "the file bootstrap-kubelet.conf has stored in $PATH_KUBERNETES directory."
+}
+
 function revert() {
     echo "exec(1/5): update kubeadm.cfg..."
     if [ ! -f "$PATH_KUBEADM_CONFIG/kubeadm.cfg" ]; then
@@ -117,9 +176,26 @@ function revert() {
 
 
     echo "exec(4/5): execute join cmd...."
-    kubeadm join --config "$PATH_FILE_TMP/kubeadm.cfg.current"
-    if [ $? -ne 0 ]; then
-        exit 1
+    if [ -z "$USE_KUBEADM" ]; then
+      # if "USE_KUBEADM is not set, default set to true"
+      export USE_KUBEADM=true
+    fi
+    if [ "$USE_KUBEADM" = true ]; then
+       echo "use kubeadm to join node to host"
+       kubeadm join --config "$PATH_FILE_TMP/kubeadm.cfg.current"
+       if [ $? -ne 0 ]; then
+          exit 1
+       fi
+    else
+       echo "NONONO use kubeadm to join node to host"
+       get_ca_certificate $JOIN_HOST
+       create_kubelet_bootstrap_config $JOIN_HOST $JOIN_TOKEN
+       if [ -f "${PATH_FILE_TMP}/kubeadm-flags.env.origin" ]; then
+          cp "${PATH_FILE_TMP}/kubeadm-flags.env.origin" "${PATH_KUBELET_LIB}" && \
+          mv "${PATH_KUBELET_LIB}/kubeadm-flags.env.origin" "${PATH_KUBELET_LIB}/kubeadm-flags.env"
+       else
+          cp "${PATH_FILE_TMP}/kubeadm-flags.env" "${PATH_KUBELET_LIB}"
+       fi
     fi
 
     echo "exec(5/5): restart cotnainerd...."
@@ -128,6 +204,24 @@ function revert() {
         exit 1
     fi
 
+    if [ "$USE_KUBEADM" = false ]; then
+           systemctl start kubelet
+           elapsed_time=0
+
+           while [ $elapsed_time -lt $KUBELET_CONF_TIMEOUT ]; do
+             if [ -f "${PATH_KUBERNETES}/${KUBELET_KUBE_CONFIG_NAME}" ]; then
+                rm -f "${PATH_KUBERNETES}/bootstrap-kubelet.conf"
+                echo "Deleted bootstrap-kubelet.conf file as kubelet.conf exists."
+                break
+             fi
+             sleep 2
+             elapsed_time=$((elapsed_time + 2))
+           done
+
+           if [ $elapsed_time -ge $KUBELET_CONF_TIMEOUT ]; then
+              echo "Timeout: kubelet.conf was not generated within $KUBELET_CONF_TIMEOUT seconds. Continuing script execution."
+           fi
+        fi
     afterRevert
     if [ $? -ne 0 ]; then
         exit 1
@@ -219,6 +313,10 @@ function check() {
     fi
 
     echo "check(2/2): copy  kubeadm-flags.env  to create $PATH_FILE_TMP , remove args[cloud-provider] and taints"
+    # Since this function is used both to detach nodes, we need to make sure we haven't copied kubeadm-flags.env before
+    if [ ! -f "${PATH_FILE_TMP}/kubeadm-flags.env.origin" ]; then
+       cp "${PATH_KUBELET_LIB}/kubeadm-flags.env" "${PATH_FILE_TMP}/kubeadm-flags.env.origin"
+    fi
     sed -e "s| --cloud-provider=external | |g" -e "w ${PATH_FILE_TMP}/kubeadm-flags.env" "$PATH_KUBELET_LIB/kubeadm-flags.env"
     sed -i "s| --register-with-taints=node.kosmos.io/unschedulable:NoSchedule||g" "${PATH_FILE_TMP}/kubeadm-flags.env"
     if [ $? -ne 0 ]; then

hack/k8s-in-k8s/port_check.sh

@@ -4,7 +4,12 @@ function check_port {
     local ip=$1
     local port=$2
 
-    if timeout 1 curl -s --connect-timeout 3 $ip:$port >/dev/null; then
+    # Check if the IP address is IPv6, then enclose it in square brackets
+    if [[ $ip =~ .*:.* ]]; then
+         ip="[$ip]"
+    fi
+
+    if timeout 1 curl -s --connect-timeout 3 "${ip}:${port}" >/dev/null; then
         return 0
     else
         return 1

pkg/kubenest/controller/endpoints.sync.controller/apiserver_external_sync_controller.go

@@ -57,12 +57,12 @@ func (e *ApiServerExternalSyncController) SyncApiServerExternalEPS(ctx context.C
 	kubeEndpoints, err := k8sClient.CoreV1().Endpoints(constants.DefaultNs).Get(ctx, "kubernetes", metav1.GetOptions{})
 	if err != nil {
 		klog.Errorf("Error getting endpoints: %v", err)
-		return fmt.Errorf("failed to get endpoints for kubernetes service: %v", err)
+		return err
 	} else {
-		klog.Infof("Endpoints for service 'kubernetes': %v", kubeEndpoints)
+		klog.V(4).Infof("Endpoints for service 'kubernetes': %v", kubeEndpoints)
 		for _, subset := range kubeEndpoints.Subsets {
 			for _, address := range subset.Addresses {
-				klog.Infof("IP: %s", address.IP)
+				klog.V(4).Infof("IP: %s", address.IP)
 			}
 		}
 	}
@@ -72,39 +72,42 @@ func (e *ApiServerExternalSyncController) SyncApiServerExternalEPS(ctx context.C
 	}
 
 	if kubeEndpoints.Subsets[0].Addresses == nil || len(kubeEndpoints.Subsets[0].Addresses) == 0 {
-		return fmt.Errorf("eps %s Addresses length is nil", "kubernetes")
+		klog.Errorf("eps %s Addresses length is nil", "kubernetes")
+		return err
 	}
 
 	apiServerExternalEndpoints, err := k8sClient.CoreV1().Endpoints(constants.DefaultNs).Get(ctx, constants.ApiServerExternalService, metav1.GetOptions{})
 	if err != nil && !apierrors.IsNotFound(err) {
-		return fmt.Errorf("failed to get endpoints for %s : %v", constants.ApiServerExternalService, err)
+		klog.Errorf("failed to get endpoints for %s : %v", constants.ApiServerExternalService, err)
+		return err
 	}
 
 	updateEPS := apiServerExternalEndpoints.DeepCopy()
 
 	if apiServerExternalEndpoints != nil {
-		klog.Infof("apiServerExternalEndpoints: %v", apiServerExternalEndpoints)
+		klog.V(4).Infof("apiServerExternalEndpoints: %v", apiServerExternalEndpoints)
 	} else {
-		klog.Info("apiServerExternalEndpoints is nil")
+		klog.V(4).Info("apiServerExternalEndpoints is nil")
 	}
 
 	if updateEPS != nil {
-		klog.Infof("updateEPS: %v", updateEPS)
+		klog.V(4).Infof("updateEPS: %v", updateEPS)
 	} else {
-		klog.Info("updateEPS is nil")
+		klog.V(4).Info("updateEPS is nil")
 	}
 
 	if len(updateEPS.Subsets) == 1 && len(updateEPS.Subsets[0].Addresses) == 1 {
 		ip := kubeEndpoints.Subsets[0].Addresses[0].IP
-		klog.Infof("IP address: %s", ip)
+		klog.V(4).Infof("IP address: %s", ip)
 		updateEPS.Subsets[0].Addresses[0].IP = ip
 
 		if _, err := k8sClient.CoreV1().Endpoints(constants.DefaultNs).Update(ctx, updateEPS, metav1.UpdateOptions{}); err != nil {
-			return fmt.Errorf("failed to update endpoints for api-server-external-service: %v", err)
+			klog.Errorf("failed to update endpoints for api-server-external-service: %v", err)
+			return err
 		}
 	} else {
 		klog.ErrorS(err, "Unexpected format of endpoints for api-server-external-service", "endpoint_data", updateEPS)
-		return fmt.Errorf("unexpected format of endpoints for api-server-external-service")
+		return err
 	}
 
 	return nil
@@ -137,7 +140,7 @@ func (e *ApiServerExternalSyncController) Reconcile(ctx context.Context, request
 		return reconcile.Result{}, nil
 	}
 
-	if targetVirtualCluster.Status.Phase != v1alpha1.AllNodeReady && targetVirtualCluster.Status.Phase != v1alpha1.Completed {
+	if targetVirtualCluster.Status.Phase != v1alpha1.Initialized {
 		return reconcile.Result{RequeueAfter: utils.DefaultRequeueTime}, nil
 	}
 

pkg/kubenest/controlplane/endpoint.go

@@ -33,7 +33,7 @@ func EnsureApiServerExternalEndPoint(dynamicClient dynamic.Interface) error {
 }
 
 func installApiServerExternalEndpointInVirtualCluster(dynamicClient dynamic.Interface) error {
-	klog.Info("begin to get kubernetes endpoint")
+	klog.V(4).Info("begin to get kubernetes endpoint")
 	kubeEndpointUnstructured, err := dynamicClient.Resource(schema.GroupVersionResource{
 		Group:    "",
 		Version:  "v1",
@@ -74,7 +74,7 @@ func installApiServerExternalEndpointInVirtualCluster(dynamicClient dynamic.Inte
 			klog.Error("create api-server-external-service endpoint failed", err)
 			return errors.Wrap(err, "failed to create api-server-external-service endpoint")
 		} else {
-			klog.Info("success create api-server-external-service endpoint:", createResult)
+			klog.V(4).Info("success create api-server-external-service endpoint:", createResult)
 		}
 	} else {
 		return errors.New("kubernetes endpoint does not exist")
@@ -110,7 +110,7 @@ func installApiServerExternalServiceInVirtualCluster(dynamicClient dynamic.Inter
 }
 
 func getEndPointPort(dynamicClient dynamic.Interface) (int32, error) {
-	klog.Info("begin to get Endpoints ports...")
+	klog.V(4).Info("begin to get Endpoints ports...")
 	endpointsRes := dynamicClient.Resource(schema.GroupVersionResource{
 		Group:    "",
 		Version:  "v1",
@@ -153,6 +153,6 @@ func getEndPointPort(dynamicClient dynamic.Interface) (int32, error) {
 		return 0, fmt.Errorf("port field not found or error parsing it")
 	}
 
-	klog.Infof("The port number was successfully obtained: %d", portNum)
+	klog.V(4).Infof("The port number was successfully obtained: %d", portNum)
 	return int32(portNum), nil
 }