feat: install yurthub component on edge nodes depending on StaticPod …

…cr resource (#1398) * feat: install yurthub component on edge nodes depending on StaticPod cr resource Signed-off-by: ricky <yricky509@gmail.com> * specify namespace Signed-off-by: ricky <yricky509@gmail.com> --------- Signed-off-by: ricky <yricky509@gmail.com>
openyurtio · Apr 20, 2023 · 801b25b · 801b25b
1 parent 8462c8d
commit 801b25b
Show file tree

Hide file tree

Showing 9 changed files with 170 additions and 19 deletions.
diff --git a/charts/openyurt/templates/yurthub-staticpod.yaml b/charts/openyurt/templates/yurthub-staticpod.yaml
@@ -0,0 +1,104 @@
+apiVersion: apps.openyurt.io/v1alpha1
+kind: StaticPod
+metadata:
+  name: yurt-hub
+  namespace: {{ include "openyurt.namespace" . }}
+spec:
+  staticPodManifest: yurt-hub
+  template:
+    metadata:
+      labels:
+        k8s-app: yurt-hub
+      name: yurt-hub
+      namespace: {{ include "openyurt.namespace" . }}
+    spec:
+      volumes:
+        - name: hub-dir
+          hostPath:
+            path: /var/lib/yurthub
+            type: DirectoryOrCreate
+        - name: kubernetes
+          hostPath:
+            path: /etc/kubernetes
+            type: Directory
+        - name: pem-dir
+          hostPath:
+            path: /var/lib/kubelet/pki
+            type: Directory
+      containers:
+        - name: yurt-hub
+          image: "{{ .Values.yurtHub.Image.registry }}/{{ .Values.yurtHub.Image.repository }}:{{ .Values.yurtHub.Image.tag }}"
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - name: hub-dir
+              mountPath: /var/lib/yurthub
+            - name: kubernetes
+              mountPath: /etc/kubernetes
+            - name: pem-dir
+              mountPath: /var/lib/kubelet/pki
+          command:
+            - yurthub
+            - --v=2
+            - --bind-address={{ .Values.yurtHub.yurthubServerAddr | quote }}
+            - --server-addr={{`{{.kubernetesServerAddr}}`}}
+            - --node-name=$(NODE_NAME)
+            - --bootstrap-file={{ .Values.yurtHub.bootstrapFile | quote }}
+            - --working-mode={{`{{.workingMode}}`}}
+            - --namespace={{ include "openyurt.namespace" . }}
+              {{`{{if .organizations }}`}}
+            - --hub-cert-organizations={{`{{.organizations}}`}}
+              {{`{{end}}`}}
+          livenessProbe:
+            httpGet:
+              host: {{ .Values.yurtHub.yurthubServerAddr | quote }}
+              path: /v1/healthz
+              port: 10267
+            initialDelaySeconds: 300
+            periodSeconds: 5
+            failureThreshold: 3
+          resources:
+            requests:
+              cpu: 150m
+              memory: 150Mi
+            limits:
+              memory: 300Mi
+          securityContext:
+            capabilities:
+              add: [ "NET_ADMIN", "NET_RAW" ]
+          env:
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+      hostNetwork: true
+      priorityClassName: system-node-critical
+      priority: 2000001000
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: read-static-pod-cm
+rules:
+  - apiGroups:
+    - ""
+    resources:
+    - configmaps
+    resourceNames:
+    - static-pod-cm-{{ include "openyurt.namespace" . }}-yurt-hub
+    verbs:
+    - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: read-static-pod-cm-bootstrap
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: read-static-pod-cm
+subjects:
+  - apiGroup: rbac.authorization.k8s.io
+    kind: Group
+    name: system:bootstrappers
+
+
diff --git a/charts/openyurt/values.yaml b/charts/openyurt/values.yaml
@@ -37,7 +37,12 @@ poolCoordinator:
 
 yurtHub:
   cacheAgents: ""
-
+  bootstrapFile: "/var/lib/yurthub/bootstrap-hub.conf"
+  yurthubServerAddr: "127.0.0.1"
+  Image:
+    registry: registry.cn-hangzhou.aliyuncs.com/openyurt
+    repository: yurthub
+    tag: latest
 yurtManager:
   # settings for log print
   log:

diff --git a/pkg/controller/staticpod/staticpod_controller.go b/pkg/controller/staticpod/staticpod_controller.go
@@ -272,20 +272,6 @@ func (r *ReconcileStaticPod) Reconcile(_ context.Context, request reconcile.Requ
 		upgradedNumber int32
 	)
 
-	// The later upgrade operation is conducted based on upgradeInfos
-	upgradeInfos, err := upgradeinfo.New(r.Client, instance, UpgradeWorkerPodPrefix)
-	if err != nil {
-		klog.Errorf(Format("Fail to get static pod and worker pod upgrade info for nodes of StaticPod %v, %v",
-			request.NamespacedName, err))
-		return ctrl.Result{}, err
-	}
-	totalNumber = int32(len(upgradeInfos))
-	// There are no nodes running target static pods in the cluster
-	if totalNumber == 0 {
-		klog.Infof(Format("No static pods need to be upgraded of StaticPod %v", request.NamespacedName))
-		return r.updateStaticPodStatus(instance, totalNumber, totalNumber, totalNumber)
-	}
-
 	// The latest hash value for static pod spec
 	// This hash value is used in three places
 	// 1. Automatically added to the annotation of static pods to facilitate checking if the running static pods are up-to-date
@@ -307,6 +293,20 @@ func (r *ReconcileStaticPod) Reconcile(_ context.Context, request reconcile.Requ
 		return ctrl.Result{}, err
 	}
 
+	// The later upgrade operation is conducted based on upgradeInfos
+	upgradeInfos, err := upgradeinfo.New(r.Client, instance, UpgradeWorkerPodPrefix)
+	if err != nil {
+		klog.Errorf(Format("Fail to get static pod and worker pod upgrade info for nodes of StaticPod %v, %v",
+			request.NamespacedName, err))
+		return ctrl.Result{}, err
+	}
+	totalNumber = int32(len(upgradeInfos))
+	// There are no nodes running target static pods in the cluster
+	if totalNumber == 0 {
+		klog.Infof(Format("No static pods need to be upgraded of StaticPod %v", request.NamespacedName))
+		return r.updateStaticPodStatus(instance, totalNumber, totalNumber, totalNumber)
+	}
+
 	// Complete upgrade info
 	{
 		// Count the number of upgraded nodes

diff --git a/pkg/yurtadm/cmd/join/join.go b/pkg/yurtadm/cmd/join/join.go
@@ -46,6 +46,7 @@ type joinOptions struct {
 	organizations            string
 	pauseImage               string
 	yurthubImage             string
+	namespace                string
 	caCertHashes             []string
 	unsafeSkipCAVerification bool
 	ignorePreflightErrors    []string
@@ -62,6 +63,7 @@ func newJoinOptions() *joinOptions {
 		criSocket:                yurtconstants.DefaultDockerCRISocket,
 		pauseImage:               yurtconstants.PauseImagePath,
 		yurthubImage:             fmt.Sprintf("%s/%s:%s", yurtconstants.DefaultOpenYurtImageRegistry, yurtconstants.Yurthub, yurtconstants.DefaultOpenYurtVersion),
+		namespace:                yurtconstants.YurthubNamespace,
 		caCertHashes:             make([]string, 0),
 		unsafeSkipCAVerification: false,
 		ignorePreflightErrors:    make([]string, 0),
@@ -118,6 +120,10 @@ func addJoinConfigFlags(flagSet *flag.FlagSet, joinOptions *joinOptions) {
 		&joinOptions.nodeName, yurtconstants.NodeName, joinOptions.nodeName,
 		`Specify the node name. if not specified, hostname will be used.`,
 	)
+	flagSet.StringVar(
+		&joinOptions.namespace, yurtconstants.Namespace, joinOptions.namespace,
+		`Specify the namespace of the yurthub staticpod configmap, if not specified, the namespace will be default.`,
+	)
 	flagSet.StringVar(
 		&joinOptions.criSocket, yurtconstants.NodeCRISocket, joinOptions.criSocket,
 		"Path to the CRI socket to connect",
@@ -202,6 +208,7 @@ type joinData struct {
 	organizations            string
 	pauseImage               string
 	yurthubImage             string
+	yurthubTemplate          string
 	kubernetesVersion        string
 	caCertHashes             []string
 	nodeLabels               map[string]string
@@ -311,6 +318,15 @@ func newJoinData(args []string, opt *joinOptions) (*joinData, error) {
 	data.kubernetesVersion = k8sVersion
 	klog.Infof("node join data info: %#+v", *data)
 
+	// get the yurthub template from the staticpod cr
+	yurthubTemplate, err := yurtadmutil.GetYurthubTemplateFromStaticPod(client, opt.namespace)
+	if err != nil {
+		klog.Errorf("failed to get yurthub template, %v", err)
+		return nil, err
+	}
+	data.yurthubTemplate = yurthubTemplate
+	klog.Infof("yurthub template: %s", yurthubTemplate)
+
 	return data, nil
 }
 
@@ -339,6 +355,11 @@ func (j *joinData) YurtHubServer() string {
 	return j.yurthubServer
 }
 
+// YurtHubTemplate returns the YurtHub template.
+func (j *joinData) YurtHubTemplate() string {
+	return j.yurthubTemplate
+}
+
 // KubernetesVersion returns the kubernetes version.
 func (j *joinData) KubernetesVersion() string {
 	return j.kubernetesVersion

diff --git a/pkg/yurtadm/cmd/join/join_test.go b/pkg/yurtadm/cmd/join/join_test.go
@@ -51,6 +51,7 @@ func TestNewJoinOptions(t *testing.T) {
 				criSocket:                yurtconstants.DefaultDockerCRISocket,
 				pauseImage:               yurtconstants.PauseImagePath,
 				yurthubImage:             fmt.Sprintf("%s/%s:%s", yurtconstants.DefaultOpenYurtImageRegistry, yurtconstants.Yurthub, yurtconstants.DefaultOpenYurtVersion),
+				namespace:                yurtconstants.YurthubNamespace,
 				caCertHashes:             make([]string, 0),
 				unsafeSkipCAVerification: false,
 				ignorePreflightErrors:    make([]string, 0),

diff --git a/pkg/yurtadm/cmd/join/joindata/data.go b/pkg/yurtadm/cmd/join/joindata/data.go
@@ -35,6 +35,7 @@ type YurtJoinData interface {
 	PauseImage() string
 	YurtHubImage() string
 	YurtHubServer() string
+	YurtHubTemplate() string
 	KubernetesVersion() string
 	TLSBootstrapCfg() *clientcmdapi.Config
 	BootstrapClient() *clientset.Clientset

diff --git a/pkg/yurtadm/constants/constants.go b/pkg/yurtadm/constants/constants.go
@@ -37,6 +37,8 @@ const (
 	DefaultCertificatesDir   = "/etc/kubernetes/pki"
 	DefaultDockerCRISocket   = "/var/run/dockershim.sock"
 	YurthubYamlName          = "yurt-hub.yaml"
+	YurthubStaticPodManifest = "yurt-hub"
+	YurthubNamespace         = "kube-system"
 	// ManifestsSubDirName defines directory name to store manifests
 	ManifestsSubDirName = "manifests"
 	// KubeletKubeConfigFileName defines the file name for the kubeconfig that the control-plane kubelet will use for talking
@@ -93,6 +95,8 @@ const (
 	TokenDiscoveryCAHash = "discovery-token-ca-cert-hash"
 	// TokenDiscoverySkipCAHash flag instruct kubeadm to skip CA hash verification (for token-based discovery)
 	TokenDiscoverySkipCAHash = "discovery-token-unsafe-skip-ca-verification"
+	// Namespace flag sets the namespace of yurthub staticpod manifest.
+	Namespace = "namespace"
 	// YurtHubImage flag sets the yurthub image for worker node.
 	YurtHubImage = "yurthub-image"
 	// YurtHubServerAddr flag set the address of yurthub server (not proxy server!)

diff --git a/pkg/yurtadm/util/kubernetes/kubernetes.go b/pkg/yurtadm/util/kubernetes/kubernetes.go
@@ -38,6 +38,7 @@ import (
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 	"k8s.io/klog/v2"
 
+	spctrlutil "github.com/openyurtio/openyurt/pkg/controller/staticpod/util"
 	"github.com/openyurtio/openyurt/pkg/projectinfo"
 	kubeconfigutil "github.com/openyurtio/openyurt/pkg/util/kubeconfig"
 	"github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/util/apiclient"
@@ -488,3 +489,20 @@ func CheckKubeletStatus() error {
 	}
 	return nil
 }
+
+// GetYurthubTemplateFromStaticPod get yurthub template from static pod
+func GetYurthubTemplateFromStaticPod(client kubernetes.Interface, namespace string) (string, error) {
+	configMap, err := apiclient.GetConfigMapWithRetry(
+		client,
+		namespace,
+		spctrlutil.WithConfigMapPrefix(spctrlutil.Hyphen(namespace, constants.YurthubStaticPodManifest)))
+	if err != nil {
+		return "", pkgerrors.Wrap(err, "failed to get yurt-hub static-pod configmap")
+	}
+	data := configMap.Data[constants.YurthubStaticPodManifest]
+	if len(data) == 0 {
+		return "", fmt.Errorf("empty manifest in configmap %v",
+			spctrlutil.WithConfigMapPrefix(spctrlutil.Hyphen(metav1.NamespaceSystem, constants.YurthubStaticPodManifest)))
+	}
+	return data, nil
+}
diff --git a/pkg/yurtadm/util/yurthub/yurthub.go b/pkg/yurtadm/util/yurthub/yurthub.go
@@ -61,14 +61,11 @@ func AddYurthubStaticYaml(data joindata.YurtJoinData, podManifestPath string) er
 
 	ctx := map[string]string{
 		"kubernetesServerAddr": kubernetesServerAddrs,
-		"image":                data.YurtHubImage(),
-		"bootstrapFile":        constants.YurtHubBootstrapConfig,
 		"workingMode":          data.NodeRegistration().WorkingMode,
 		"organizations":        data.NodeRegistration().Organizations,
-		"yurthubServerAddr":    data.YurtHubServer(),
 	}
 
-	yurthubTemplate, err := templates.SubsituteTemplate(constants.YurthubTemplate, ctx)
+	yurthubTemplate, err := templates.SubsituteTemplate(data.YurtHubTemplate(), ctx)
 	if err != nil {
 		return err
 	}