Merge pull request #56020 from xiangpengzhao/write-kubeletconf-todisk

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

 Write marshalled kubeletconfig object to init-config-dir

**What this PR does / why we need it**:
from @luxas :
>Write the the marshalled kubeletconfig object to /var/lib/kubelet/config/init/kubelet so that the kubelet will start up with the right params on init/join. The only params expected in the kubelet command-line after this is kubelet --init-config-dir /var/lib/kubelet/config/init --dynamic-config-dir /var/lib/kubelet/config/dynamic

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
ref: kubernetes/kubeadm#28 (comment)

**Special notes for your reviewer**:
/cc @kubernetes/sig-cluster-lifecycle-pr-reviews

**Release note**:

```release-note
NONE
```

cmd/kubeadm/app/cmd/BUILD

@@ -70,7 +70,6 @@ go_library(
         "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/version:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes:go_default_library",

cmd/kubeadm/app/cmd/init.go

@@ -338,6 +338,14 @@ func (i *Init) Run(out io.Writer) error {
 		return fmt.Errorf("error printing files on dryrun: %v", err)
 	}
 
+	// NOTE: flag "--dynamic-config-dir" should be specified in /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
+	if features.Enabled(i.cfg.FeatureGates, features.DynamicKubeletConfig) {
+		// Write base kubelet configuration for dynamic kubelet configuration feature.
+		if err := kubeletphase.WriteInitKubeletConfigToDiskOnMaster(i.cfg); err != nil {
+			return fmt.Errorf("error writing base kubelet configuration to disk: %v", err)
+		}
+	}
+
 	// Create a kubernetes client and wait for the API server to be healthy (if not dryrunning)
 	client, err := createClient(i.cfg, i.dryRun)
 	if err != nil {
@@ -364,7 +372,7 @@ func (i *Init) Run(out io.Writer) error {
 	if features.Enabled(i.cfg.FeatureGates, features.DynamicKubeletConfig) {
 		// Create base kubelet configuration for dynamic kubelet configuration feature.
 		if err := kubeletphase.CreateBaseKubeletConfiguration(i.cfg, client); err != nil {
-			return fmt.Errorf("error uploading configuration: %v", err)
+			return fmt.Errorf("error creating base kubelet configuration: %v", err)
 		}
 	}
 

cmd/kubeadm/app/cmd/join.go

@@ -20,7 +20,6 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"os"
 	"path/filepath"
 	"strings"
 
@@ -30,8 +29,6 @@ import (
 
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/sets"
-	"k8s.io/apimachinery/pkg/util/wait"
-	clientset "k8s.io/client-go/kubernetes"
 	certutil "k8s.io/client-go/util/cert"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1"
@@ -262,36 +259,11 @@ func (j *Join) Run(out io.Writer) error {
 
 	// NOTE: flag "--dynamic-config-dir" should be specified in /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
 	if features.Enabled(j.cfg.FeatureGates, features.DynamicKubeletConfig) {
-		client, err := getTLSBootstrappedClient()
-		if err != nil {
-			return err
-		}
-
-		// Update the node with remote base kubelet configuration
-		if err := kubeletphase.UpdateNodeWithConfigMap(client, j.cfg.NodeName); err != nil {
-			return err
+		if err := kubeletphase.ConsumeBaseKubeletConfiguration(j.cfg.NodeName); err != nil {
+			return fmt.Errorf("error consuming base kubelet configuration: %v", err)
 		}
 	}
 
 	fmt.Fprintf(out, joinDoneMsgf)
 	return nil
 }
-
-// getTLSBootstrappedClient waits for the kubelet to perform the TLS bootstrap
-// and then creates a client from config file /etc/kubernetes/kubelet.conf
-func getTLSBootstrappedClient() (clientset.Interface, error) {
-	fmt.Println("[tlsbootstrap] Waiting for the kubelet to perform the TLS Bootstrap...")
-
-	kubeletKubeConfig := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletKubeConfigFileName)
-
-	// Loop on every falsy return. Return with an error if raised. Exit successfully if true is returned.
-	err := wait.PollImmediateInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
-		_, err := os.Stat(kubeletKubeConfig)
-		return (err == nil), nil
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	return kubeconfigutil.ClientSetFromFile(kubeletKubeConfig)
-}

cmd/kubeadm/app/constants/constants.go

@@ -155,6 +155,14 @@ const (
 	// after https://github.com/kubernetes/kubernetes/pull/53833 being merged.
 	KubeletBaseConfigurationConfigMapKey = "kubelet"
 
+	// KubeletBaseConfigurationDir specifies the directory on the node where stores the initial remote configuration of kubelet
+	KubeletBaseConfigurationDir = "/var/lib/kubelet/config/init"
+
+	// KubeletBaseConfigurationFile specifies the file name on the node which stores initial remote configuration of kubelet
+	// TODO: Use the constant ("kubelet.config.k8s.io") defined in pkg/kubelet/kubeletconfig/util/keys/keys.go
+	// after https://github.com/kubernetes/kubernetes/pull/53833 being merged.
+	KubeletBaseConfigurationFile = "kubelet"
+
 	// MinExternalEtcdVersion indicates minimum external etcd version which kubeadm supports
 	MinExternalEtcdVersion = "3.0.14"
 

cmd/kubeadm/app/phases/kubelet/BUILD

@@ -10,6 +10,7 @@ go_library(
         "//cmd/kubeadm/app/constants:go_default_library",
         "//cmd/kubeadm/app/util:go_default_library",
         "//cmd/kubeadm/app/util/apiclient:go_default_library",
+        "//cmd/kubeadm/app/util/kubeconfig:go_default_library",
         "//pkg/apis/rbac/v1:go_default_library",
         "//pkg/kubelet/apis/kubeletconfig/scheme:go_default_library",
         "//pkg/kubelet/apis/kubeletconfig/v1alpha1:go_default_library",
@@ -24,20 +25,6 @@ go_library(
     ],
 )
 
-filegroup(
-    name = "package-srcs",
-    srcs = glob(["**"]),
-    tags = ["automanaged"],
-    visibility = ["//visibility:private"],
-)
-
-filegroup(
-    name = "all-srcs",
-    srcs = [":package-srcs"],
-    tags = ["automanaged"],
-    visibility = ["//visibility:public"],
-)
-
 go_test(
     name = "go_default_test",
     srcs = ["kubelet_test.go"],
@@ -54,3 +41,17 @@ go_test(
         "//vendor/k8s.io/client-go/testing:go_default_library",
     ],
 )
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

cmd/kubeadm/app/phases/kubelet/kubelet.go

@@ -19,6 +19,9 @@ package kubelet
 import (
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
 
 	"k8s.io/api/core/v1"
 	rbac "k8s.io/api/rbac/v1"
@@ -32,6 +35,7 @@ import (
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
+	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
 	rbachelper "k8s.io/kubernetes/pkg/apis/rbac/v1"
 	kubeletconfigscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme"
 	kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1"
@@ -67,11 +71,30 @@ func CreateBaseKubeletConfiguration(cfg *kubeadmapi.MasterConfiguration, client
 		return fmt.Errorf("error creating base kubelet configmap RBAC rules: %v", err)
 	}
 
-	return UpdateNodeWithConfigMap(client, cfg.NodeName)
+	return updateNodeWithConfigMap(client, cfg.NodeName)
 }
 
-// UpdateNodeWithConfigMap updates node ConfigSource with KubeletBaseConfigurationConfigMap
-func UpdateNodeWithConfigMap(client clientset.Interface, nodeName string) error {
+// ConsumeBaseKubeletConfiguration consumes base kubelet configuration for dynamic kubelet configuration feature.
+func ConsumeBaseKubeletConfiguration(nodeName string) error {
+	client, err := getLocalNodeTLSBootstrappedClient()
+	if err != nil {
+		return err
+	}
+
+	kubeletCfg, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(kubeadmconstants.KubeletBaseConfigurationConfigMap, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+
+	if err := writeInitKubeletConfigToDisk([]byte(kubeletCfg.Data[kubeadmconstants.KubeletBaseConfigurationConfigMapKey])); err != nil {
+		return fmt.Errorf("failed to write initial remote configuration of kubelet to disk for node %s: %v", nodeName, err)
+	}
+
+	return updateNodeWithConfigMap(client, nodeName)
+}
+
+// updateNodeWithConfigMap updates node ConfigSource with KubeletBaseConfigurationConfigMap
+func updateNodeWithConfigMap(client clientset.Interface, nodeName string) error {
 	fmt.Printf("[kubelet] Using Dynamic Kubelet Config for node %q; config sourced from ConfigMap %q in namespace %s",
 		nodeName, kubeadmconstants.KubeletBaseConfigurationConfigMap, metav1.NamespaceSystem)
 
@@ -148,9 +171,64 @@ func createKubeletBaseConfigMapRBACRules(client clientset.Interface) error {
 		},
 		Subjects: []rbac.Subject{
 			{
-				Kind: "Group",
+				Kind: rbac.GroupKind,
 				Name: kubeadmconstants.NodesGroup,
 			},
+			{
+				Kind: rbac.GroupKind,
+				Name: kubeadmconstants.NodeBootstrapTokenAuthGroup,
+			},
 		},
 	})
 }
+
+// getLocalNodeTLSBootstrappedClient waits for the kubelet to perform the TLS bootstrap
+// and then creates a client from config file /etc/kubernetes/kubelet.conf
+func getLocalNodeTLSBootstrappedClient() (clientset.Interface, error) {
+	fmt.Println("[tlsbootstrap] Waiting for the kubelet to perform the TLS Bootstrap...")
+
+	kubeletKubeConfig := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletKubeConfigFileName)
+
+	// Loop on every falsy return. Return with an error if raised. Exit successfully if true is returned.
+	err := wait.PollImmediateInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
+		_, err := os.Stat(kubeletKubeConfig)
+		return (err == nil), nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return kubeconfigutil.ClientSetFromFile(kubeletKubeConfig)
+}
+
+// WriteInitKubeletConfigToDiskOnMaster writes base kubelet configuration to disk on master.
+func WriteInitKubeletConfigToDiskOnMaster(cfg *kubeadmapi.MasterConfiguration) error {
+	fmt.Printf("[kubelet] Writing base configuration of kubelets to disk on master node %s", cfg.NodeName)
+
+	_, kubeletCodecs, err := kubeletconfigscheme.NewSchemeAndCodecs()
+	if err != nil {
+		return err
+	}
+
+	kubeletBytes, err := kubeadmutil.MarshalToYamlForCodecs(cfg.KubeletConfiguration.BaseConfig, kubeletconfigv1alpha1.SchemeGroupVersion, *kubeletCodecs)
+	if err != nil {
+		return err
+	}
+
+	if err := writeInitKubeletConfigToDisk(kubeletBytes); err != nil {
+		return fmt.Errorf("failed to write base configuration of kubelet to disk on master node %s: %v", cfg.NodeName, err)
+	}
+
+	return nil
+}
+
+func writeInitKubeletConfigToDisk(kubeletConfig []byte) error {
+	if err := os.MkdirAll(kubeadmconstants.KubeletBaseConfigurationDir, 0644); err != nil {
+		return fmt.Errorf("failed to create directory %q: %v", kubeadmconstants.KubeletBaseConfigurationDir, err)
+	}
+	baseConfigFile := filepath.Join(kubeadmconstants.KubeletBaseConfigurationDir, kubeadmconstants.KubeletBaseConfigurationFile)
+	if err := ioutil.WriteFile(baseConfigFile, kubeletConfig, 0644); err != nil {
+		return fmt.Errorf("failed to write initial remote configuration of kubelet into file %q: %v", baseConfigFile, err)
+	}
+	return nil
+}

cmd/kubeadm/app/phases/kubelet/kubelet_test.go

@@ -114,7 +114,7 @@ func TestUpdateNodeWithConfigMap(t *testing.T) {
 		return true, nil, nil
 	})
 
-	if err := UpdateNodeWithConfigMap(client, nodeName); err != nil {
+	if err := updateNodeWithConfigMap(client, nodeName); err != nil {
 		t.Errorf("UpdateNodeWithConfigMap: unexepected error %v", err)
 	}
 }