diff --git a/config/yurtctl-servant/Dockerfile b/config/yurtctl-servant/Dockerfile index fd1f3963cfa..1154c8492cd 100644 --- a/config/yurtctl-servant/Dockerfile +++ b/config/yurtctl-servant/Dockerfile @@ -1,4 +1 @@ -FROM alpine:3.8 - -RUN mkdir -p /var/lib/openyurt -ADD setup_edgenode /var/lib/openyurt +FROM alpine:3.8 \ No newline at end of file diff --git a/config/yurtctl-servant/setup_edgenode b/config/yurtctl-servant/setup_edgenode deleted file mode 100755 index f108e677392..00000000000 --- a/config/yurtctl-servant/setup_edgenode +++ /dev/null @@ -1,238 +0,0 @@ -#!/usr/bin/env bash -# This script can not be executed directly, it is baked in the -# openyurt/yurtctl-servant, before exeuction, context value (i.e., __variable__) -# need to be replaced based on the environment variables set in the pod, -# and will be executed as a subprogram of the nsenter command. - -set -o errexit -set -o pipefail - -KUBELET_SVC=${KUBELET_SVC:-/etc/systemd/system/kubelet.service.d/10-kubeadm.conf} -OPENYURT_DIR=${OPENYURT_DIR:-/var/lib/openyurt} -STATIC_POD_PATH=${STATIC_POD_PATH:-/etc/kubernetes/manifests} -ACTION=$1 - -# PROVIDER can be nounset -set -o nounset - -declare -r YURTHUB_TEMPLATE=' -apiVersion: v1 -kind: Pod -metadata: - labels: - k8s-app: yurt-hub - name: yurt-hub - namespace: kube-system -spec: - volumes: - - name: hub-dir - hostPath: - path: /var/lib/yurthub - type: DirectoryOrCreate - - name: kubernetes - hostPath: - path: /etc/kubernetes - type: Directory - containers: - - name: yurt-hub - image: __yurthub_image__ - imagePullPolicy: IfNotPresent - volumeMounts: - - name: hub-dir - mountPath: /var/lib/yurthub - - name: kubernetes - mountPath: /etc/kubernetes - command: - - yurthub - - --v=2 - - --server-addr=__kubernetes_service_addr__ - - --node-name=$(NODE_NAME) - - --join-token=__join_token__ - livenessProbe: - httpGet: - host: 127.0.0.1 - path: /v1/healthz - port: 10261 - initialDelaySeconds: 300 - periodSeconds: 5 - failureThreshold: 3 - resources: - requests: - cpu: 150m - memory: 150Mi - limits: - memory: 300Mi - securityContext: - capabilities: - add: ["NET_ADMIN", "NET_RAW"] - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - hostNetwork: true - priorityClassName: system-node-critical - priority: 2000001000 -' - -# log outputs the log message with date and program prefix -log() { - echo "$(date +"%m/%d/%Y-%T-%Z") [YURT_SERVANT] [LOG] $@" -} - -# error outputs the error message with data program prefix -error() { - echo "$(date +"%m/%d/%Y-%T-%Z") [YURT_SERVANT] [ERROR] $@" -} - -check_addr() -{ - echo $1|grep -E '^(http(s)?:\/\/)?[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+(:[0-9]{1,5})?$' > /dev/null; - if [ $? -ne 0 ] - then - log "apiserver addr $1 is error." - exit 1 - fi - - log "apiserver addr $1 is ok." - return 0 -} - -# preset creates the /var/lib/kubelet/pki/kubelet-client-current.pem if -# it does not exist -preset() { - # if $KUBELET_CLI_PEM doesn't exist, create one based on 'client-certificate-data' - # and 'client-key-data' of $KUBELET_CONF - if [ ! -f $KUBELET_SVC ]; then - log "$KUBELET_SVC don't exist." - exit 1 - fi - if [ ! -d $STATIC_POD_PATH ]; then - log "$STATIC_POD_PATH don't exist." - exit 1 - fi -} - -# setup_yurthub sets up the yurthub pod and wait for the its status to be Running -setup_yurthub() { - # 1. put yurt-hub yaml into /etc/kubernetes/manifests - log "setting up yurthub on nodes" - KUBELET_CONF=`cat $KUBELET_SVC | grep -Eo '\-\-kubeconfig=.*kubelet.conf' | awk -F '=' '{print $2}'` - apiserver_addr=`cat $KUBELET_CONF | grep "server:" | awk '{print $2}'` - check_addr $apiserver_addr - log "setting up yurthub apiserver addr ${apiserver_addr}." - yurthub_yaml=$(echo "$YURTHUB_TEMPLATE" | - sed "s|__kubernetes_service_addr__|${apiserver_addr}|") - - echo "$yurthub_yaml" > ${STATIC_POD_PATH}/yurt-hub.yaml - log "create the ${STATIC_POD_PATH}/yurt-hub.yaml" - # 2. wait yurthub pod to be ready - local retry=5 - while [ $retry -ge 0 ] - do - sleep 10 - # NOTE: context variables need to be replaced before exeuction - set +e - local hub_healthz - hub_healthz=$(netstat -nlutp | grep 10261 | wc -l) - set -e - - if [ "$hub_healthz" == "1" ]; then - log "yurt-hub-$NODE_NAME healthz is OK" - return - else - retry=$((retry-1)) - if [ $retry -ge 0 ]; then - log "yurt-hub-$NODE_NAME is not ready, will retry $retry times" - else - error "yurt-hub-$NODE_NAME failed, after retry 5 times" - exit 1 - fi - continue - fi - done -} - -# reset_kubelet changes the configuration of the kubelet service and restart it -reset_kubelet() { - # 1. create a working dir to store revised kubelet.conf - mkdir -p $OPENYURT_DIR - cp $KUBELET_CONF $OPENYURT_DIR/ - # 2. revise the copy of the kubelet.conf - cat << EOF > $OPENYURT_DIR/kubelet.conf -apiVersion: v1 -clusters: -- cluster: - server: http://127.0.0.1:10261 - name: default-cluster -contexts: -- context: - cluster: default-cluster - namespace: default - user: default-auth - name: default-context -current-context: default-context -kind: Config -preferences: {} -EOF - log "revised kubeconfig $OPENYURT_DIR/kubelet.conf is generated" - # 3. revise the kubelet.service drop-in - # 3.1 make a backup for the origin kubelet.service - cp $KUBELET_SVC ${KUBELET_SVC}.bk - # 3.2 revise the drop-in, point it to the $OPENYURT_DIR/kubelet.conf - sed -i "s/--bootstrap.*bootstrap-kubelet.conf//g; - s|--kubeconfig=.*kubelet.conf|--kubeconfig=$OPENYURT_DIR\/kubelet.conf|g" $KUBELET_SVC - log "kubelet.service drop-in file is revised" - # 4. reset the kubelete.service - systemctl daemon-reload - systemctl restart kubelet.service - log "kubelet has been restarted" -} - -# remove_yurthub deletes the yurt-hub pod -remove_yurthub() { - # remove the yurt-hub.yaml to delete the yurt-hub - [ -f $STATIC_POD_PATH/yurt-hub.yaml ] && - rm $STATIC_POD_PATH/yurt-hub.yaml - log "yurt-hub has been removed" -} - -# revert_kubelet resets the kubelet service and makes it connect to the -# apiserver directly -revert_kubelet() { - # 1. remove openyurt's kubelet.conf if exist - [ -f $OPENYURT_DIR/kubelet.conf ] && rm $OPENYURT_DIR/kubelet.conf - if [ -f ${KUBELET_SVC}.bk ]; then - # if found, use the backup file - log "found backup file ${KUBELET_SVC}.bk, will use it to revert the node" - mv ${KUBELET_SVC}.bk $KUBELET_SVC - else - # if the backup file doesn't not exist, revise the kubelet.service drop-in - log "didn't find the ${KUBELET_SVC}.bk, will revise the $KUBELET_SVC directly" - exit 1 - fi - # 2. reset the kubelete.service - systemctl daemon-reload - systemctl restart kubelet.service - log "kubelet has been reset back to default" -} - -case $ACTION in - convert) - preset - setup_yurthub - reset_kubelet - ;; - revert) - revert_kubelet - remove_yurthub - ;; - *) - error "unknown action $ACTION" - exit 1 - ;; -esac - - - -log "done" diff --git a/pkg/yurtctl/cmd/convert/convert.go b/pkg/yurtctl/cmd/convert/convert.go index e4e40208d24..10b02553882 100644 --- a/pkg/yurtctl/cmd/convert/convert.go +++ b/pkg/yurtctl/cmd/convert/convert.go @@ -236,13 +236,13 @@ func (co *ConvertOptions) RunConvert() (err error) { continue } // label node as edge node - //klog.Infof("mark %s as the edge-node", node.GetName()) - //edgeNodeNames = append(edgeNodeNames, node.GetName()) - //_, err = kubeutil.LabelNode(co.clientSet, - // &node, projectinfo.GetEdgeWorkerLabelKey(), "true") - //if err != nil { - // return - //} + klog.Infof("mark %s as the edge-node", node.GetName()) + edgeNodeNames = append(edgeNodeNames, node.GetName()) + _, err = kubeutil.LabelNode(co.clientSet, + &node, projectinfo.GetEdgeWorkerLabelKey(), "true") + if err != nil { + return + } } // 3. deploy yurt controller manager @@ -317,7 +317,6 @@ func (co *ConvertOptions) RunConvert() (err error) { } if err = kubeutil.RunServantJobs(co.clientSet, map[string]string{ "provider": string(co.Provider), - //"action": "convert", "yurtctl_servant_image": co.YurctlServantImage, "yurthub_image": co.YurhubImage, "joinToken": joinToken, diff --git a/pkg/yurtctl/cmd/convert/edgenode.go b/pkg/yurtctl/cmd/convert/edgenode.go index dedf0c4dcda..08e351d289d 100644 --- a/pkg/yurtctl/cmd/convert/edgenode.go +++ b/pkg/yurtctl/cmd/convert/edgenode.go @@ -2,45 +2,45 @@ package convert import ( "fmt" - "github.com/alibaba/openyurt/pkg/projectinfo" - "github.com/alibaba/openyurt/pkg/yurtctl/lock" - kubeutil "github.com/alibaba/openyurt/pkg/yurtctl/util/kubernetes" - "github.com/spf13/cobra" - "github.com/spf13/pflag" "io/ioutil" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/klog" "net/http" + "net/url" "os" "path/filepath" "strings" "time" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/klog" + + "github.com/alibaba/openyurt/pkg/projectinfo" + "github.com/alibaba/openyurt/pkg/yurtctl/lock" enutil "github.com/alibaba/openyurt/pkg/yurtctl/util/edgenode" + kubeutil "github.com/alibaba/openyurt/pkg/yurtctl/util/kubernetes" + "github.com/alibaba/openyurt/pkg/yurthub/healthchecker" ) const ( - kubeletConfRegularExpression = "\\-\\-kubeconfig=.*kubelet.conf" + kubeletConfigRegularExpression = "\\-\\-kubeconfig=.*kubelet.conf" apiserverAddrRegularExpression = "server: (http(s)?:\\/\\/)?[\\d\\D][\\d\\D]{0,62}(\\.[\\d\\D][-\\d\\D]{0,62})+(:[\\d]{1,5})?$" - checkFrequency = 10 * time.Second - filemode = 0666 - dirmode = 0755 + hubHealthzCheckFrequency = 10 * time.Second + failedRetry = 5 + filemode = 0666 + dirmode = 0755 ) // EdgenodeOptions holds the command-line options for 'convert edgenode' sub command type ConvertEdgeNodeOptions struct { - clientSet *kubernetes.Clientset - //Provider Provider - YurthubImage string - //YurttunnelAgentImage string - //DeployTunnel bool - JoinToken string - StaticPodPath string - - kubeletService string - openyurtDir string + clientSet *kubernetes.Clientset + YurthubImage string + JoinToken string + PodMainfestPath string + kubeletSvcPath string + openyurtDir string } func NewConvertEdgeNodeOptions() *ConvertEdgeNodeOptions { @@ -48,112 +48,79 @@ func NewConvertEdgeNodeOptions() *ConvertEdgeNodeOptions { } func NewConvertEdgeNodeCmd() *cobra.Command { - ceo := NewConvertEdgeNodeOptions() + c := NewConvertEdgeNodeOptions() cmd := &cobra.Command{ - Use: "edgenode", - Short: "Converts the kubernetes node to a yurt node", + Use: "edgenode", + Short: "Converts the kubernetes node to a yurt node", Run: func(cmd *cobra.Command, _ []string) { - if err := ceo.Complete(cmd.Flags()); err != nil { + if err := c.Complete(cmd.Flags()); err != nil { klog.Fatalf("fail to complete the convert edgenode option: %s", err) } - //if err := ceo.Vaildate(); err != nil { - // klog.Fatalf("convert edgenode option is invaild: %s", err) - //} - if err := ceo.RunConvertEdgeNode(); err != nil { + if err := c.RunConvertEdgeNode(); err != nil { klog.Fatalf("fail to covert the kubernetes node to a yurt node: %s", err) } }, } - //cmd.Flags().StringP("provider", "p", "minikube", "The provider of the original Kubernetes cluster.") cmd.Flags().String("join-token", "", "the join token.") cmd.Flags().String("yurthub-image", "openyurt/yurthub:latest", "the yurthub image.") - //cmd.Flags().String("yurt-tunnel-agent-image", "openyurt/yurt-tunnel-agent:latest", "The yurt-tunnel-agent image.") - //cmd.Flags().BoolP("deploy-yurttunnel", "t", false, "if set, yurttunnel will be deployed.") cmd.Flags().String("pod-manifest-path", "/etc/kubernetes/manifests", "Path to the directory on edge node containing static pod files.") return cmd } -func (ceo *ConvertEdgeNodeOptions) Complete(flags *pflag.FlagSet) error { - //pStr, err := flags.GetString("provider") - //if err != nil { - // return err - //} - //ceo.Provider = Provider(pStr) - - yhi, err := flags.GetString("yurthub-image") +func (c *ConvertEdgeNodeOptions) Complete(flags *pflag.FlagSet) error { + yurthubImage, err := flags.GetString("yurthub-image") if err != nil { return err } - ceo.YurthubImage = yhi - - //ytai, err := flags.GetString("yurt-tunnel-agent-image") - //if err != nil { - // return err - //} - //ceo.YurttunnelAgentImage = ytai + c.YurthubImage = yurthubImage - //dt, err := flags.GetBool("deploy-yurttunnel") - //if err != nil { - // return err - //} - //ceo.DeployTunnel = dt - - ceo.clientSet, err = kubeutil.GenClientSet(flags) + joinToken, err := flags.GetString("join-token") if err != nil { return err } + c.JoinToken = joinToken - jtStr, err := flags.GetString("join-token") + podMainfestPath, err := flags.GetString("pod-manifest-path") if err != nil { return err } - ceo.JoinToken = jtStr + c.PodMainfestPath = podMainfestPath - spp, err := flags.GetString("pod-manifest-path") + c.clientSet, err = kubeutil.GenClientSet(flags) if err != nil { return err } - ceo.StaticPodPath = spp - kls := os.Getenv("KUBELET_SVC") - if kls == "" { - kls = enutil.KubeletSvc + kubeletSvcPath := os.Getenv("KUBELET_SVC") + if kubeletSvcPath == "" { + kubeletSvcPath = enutil.KubeletSvcPath } - ceo.kubeletService = kls + c.kubeletSvcPath = kubeletSvcPath - oyd := os.Getenv("OPENYURT_DIR") - if oyd == "" { - oyd = enutil.OpenyurtDir + openyurtDir := os.Getenv("OPENYURT_DIR") + if openyurtDir == "" { + openyurtDir = enutil.OpenyurtDir } - ceo.openyurtDir = oyd + c.openyurtDir = openyurtDir return nil } -//func (ceo *ConvertEdgeNodeOptions) Validate() error { -// if ceo.Provider != ProviderMinikube && -// ceo.Provider != ProviderACK && ceo.Provider != ProviderKubeadm { -// return fmt.Errorf("unknown provider: %s, valid providers are: minikube, ack", -// ceo.Provider) -// } -// return nil -//} - -func (ceo *ConvertEdgeNodeOptions) RunConvertEdgeNode() (err error) { - if err = lock.AcquireLock(ceo.clientSet); err != nil { +func (c *ConvertEdgeNodeOptions) RunConvertEdgeNode() (err error) { + if err = lock.AcquireLock(c.clientSet); err != nil { return } defer func() { - if releaseLockErr := lock.ReleaseLock(ceo.clientSet); releaseLockErr != nil { + if releaseLockErr := lock.ReleaseLock(c.clientSet); releaseLockErr != nil { klog.Error(releaseLockErr) } }() klog.V(4).Info("successfully acquire the lock") // 1. check the server version - if err = kubeutil.ValidateServerVersion(ceo.clientSet); err != nil { + if err = kubeutil.ValidateServerVersion(c.clientSet); err != nil { return } klog.V(4).Info("the server version is valid") @@ -163,51 +130,28 @@ func (ceo *ConvertEdgeNodeOptions) RunConvertEdgeNode() (err error) { if err != nil { return err } - node, err := ceo.clientSet.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) + node, err := c.clientSet.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) if err != nil { return err } - isEdgeNode, ok := node.Labels[projectinfo.GetEdgeWorkerLabelKey()] - if ok { - if isEdgeNode == "true" { - klog.Infof("the node %s is already a yurt node", nodeName) - return nil - } else { - return fmt.Errorf("the node %s is already a yurt node, and can't convert a cloud node to a edge node") - } - } klog.Infof("mark %s as the edge-node", nodeName) - _, err = kubeutil.LabelNode(ceo.clientSet, node, projectinfo.GetEdgeWorkerLabelKey(), "true") + _, err = kubeutil.LabelNode(c.clientSet, node, projectinfo.GetEdgeWorkerLabelKey(), "true") if err != nil { return err } - // 3. deploy the yurttunnel if required - //var edgeNodeNames []string - //edgeNodeNames = append(edgeNodeNames, nodeName) - //if ceo.DeployTunnel { - // // we will deploy yurt-tunnel-agent on every edge node - // if err = deployYurttunnelAgent(ceo.clientSet, - // edgeNodeNames, - // ceo.YurttunnelAgentImage); err != nil { - // err = fmt.Errorf("fail to deploy the yurt-tunnel-agent: %s", err) - // return - // } - // klog.Info("yurt-tunnel-agent is deployed") - //} - - // 4. deploy yurt-hub and reset the kubelet service - if ceo.JoinToken == "" { - ceo.JoinToken, err = kubeutil.GetOrCreateJoinTokenString(ceo.clientSet) + // 3. deploy yurt-hub and reset the kubelet service + if c.JoinToken == "" { + c.JoinToken, err = kubeutil.GetOrCreateJoinTokenString(c.clientSet) if err != nil { return err } } - err = ceo.SetupYurthub() + err = c.SetupYurthub() if err != nil { return fmt.Errorf("fail to set up the yurthub pod: %v", err) } - err = ceo.ResetKubelet() + err = c.ResetKubelet() if err != nil { return fmt.Errorf("fail to reset the kubelet service: %v", err) } @@ -215,51 +159,51 @@ func (ceo *ConvertEdgeNodeOptions) RunConvertEdgeNode() (err error) { } // SetupYurthub sets up the yurthub pod and wait for the its status to be Running -func (ceo *ConvertEdgeNodeOptions) SetupYurthub() error { +func (c *ConvertEdgeNodeOptions) SetupYurthub() error { // 1. put yurt-hub yaml into /etc/kubernetes/manifests klog.Infof("setting up yurthub on node") // 1-1. get apiserver address - kubeletConf, err := enutil.GetSingleContentFromFile(ceo.kubeletService, kubeletConfRegularExpression) + kubeletConfPath, err := enutil.GetSingleContentFromFile(c.kubeletSvcPath, kubeletConfigRegularExpression) if err != nil { return err } - kc := strings.Split(kubeletConf, "=")[1] - apiserverAddr, err := enutil.GetSingleContentFromFile(kc, apiserverAddrRegularExpression) + kubeletConfPath = strings.Split(kubeletConfPath, "=")[1] + apiserverAddr, err := enutil.GetSingleContentFromFile(kubeletConfPath, apiserverAddrRegularExpression) if err != nil { return err } - ad := strings.Split(apiserverAddr, " ")[1] + apiserverAddr = strings.Split(apiserverAddr, " ")[1] // 1-2. replace variables in yaml file klog.Infof("setting up yurthub apiserver addr") - yurthubYaml := enutil.ReplaceRegularExpression(enutil.YurthubTemplate, + yurthubTemplate := enutil.ReplaceRegularExpression(enutil.YurthubTemplate, map[string]string{ - "__kubernetes_service_addr__": ad, - "__yurthub_image__": ceo.YurthubImage, - "__join_token__": ceo.JoinToken, + "__kubernetes_service_addr__": apiserverAddr, + "__yurthub_image__": c.YurthubImage, + "__join_token__": c.JoinToken, }) // 1-3. create yurthub.yaml - _, err = enutil.DirExists(ceo.StaticPodPath) + _, err = enutil.DirExists(c.PodMainfestPath) if err != nil { return err } - err = ioutil.WriteFile(ceo.GetYurthubYaml(), []byte(yurthubYaml), filemode) // 文件权限：https://blog.csdn.net/youngwhz1/article/details/89675137 + err = ioutil.WriteFile(c.GetYurthubYaml(), []byte(yurthubTemplate), filemode) if err != nil { return err } - klog.Infof("create the %s/yurt-hub.yaml", ceo.StaticPodPath) + klog.Infof("create the %s/yurt-hub.yaml", c.PodMainfestPath) // 2. wait yurthub pod to be ready - err = HubHealthcheck(enutil.ServerHealthzAddr) + err = HubHealthcheck() return err } -func (ceo *ConvertEdgeNodeOptions) ResetKubelet() error { +func (c *ConvertEdgeNodeOptions) ResetKubelet() error { // 1. create a working dir to store revised kubelet.conf - err := os.MkdirAll(ceo.openyurtDir, dirmode) + err := os.MkdirAll(c.openyurtDir, dirmode) if err != nil { return err } - fullpath := ceo.GetYurthubKubeletConf() + fullpath := c.GetYurthubKubeletConf() err = ioutil.WriteFile(fullpath, []byte(enutil.OpenyurtKubeletConf), filemode) if err != nil { return err @@ -268,19 +212,19 @@ func (ceo *ConvertEdgeNodeOptions) ResetKubelet() error { // 2. revise the kubelet.service drop-in // 2.1 make a backup for the origin kubelet.service - bkfile := ceo.GetKubeletSvcBackup() - err = enutil.CopyFile(ceo.kubeletService, bkfile) + bkfile := c.GetKubeletSvcBackup() + err = enutil.CopyFile(c.kubeletSvcPath, bkfile) // 2.2 revise the drop-in, point it to the $OPENYURT_DIR/kubelet.conf - contentbyte, err := ioutil.ReadFile(ceo.kubeletService) + contentbyte, err := ioutil.ReadFile(c.kubeletSvcPath) if err != nil { return err } - kubeConfigSetup := fmt.Sprintf("--kubeconfig=%s\\kubelet.conf", ceo.openyurtDir) + kubeConfigSetup := fmt.Sprintf("--kubeconfig=%s\\kubelet.conf", c.openyurtDir) content := enutil.ReplaceRegularExpression(string(contentbyte), map[string]string{ - "--bootstrap.*bootstrap-kubelet.conf": "", - "--kubeconfig=.*kubelet.conf": kubeConfigSetup, + "--bootstrap.*bootstrap-kubelet.conf": "", + "--kubeconfig=.*kubelet.conf": kubeConfigSetup, }) - err = ioutil.WriteFile(ceo.kubeletService, []byte(content), filemode) + err = ioutil.WriteFile(c.kubeletSvcPath, []byte(content), filemode) if err != nil { return err } @@ -302,26 +246,32 @@ func (ceo *ConvertEdgeNodeOptions) ResetKubelet() error { return nil } -func (ceo *ConvertEdgeNodeOptions) GetYurthubYaml() string { - return filepath.Join(ceo.StaticPodPath, enutil.YurthubYamlName) +func (c *ConvertEdgeNodeOptions) GetYurthubYaml() string { + return filepath.Join(c.PodMainfestPath, enutil.YurthubYamlName) } -func (ceo *ConvertEdgeNodeOptions)GetYurthubKubeletConf() string { - return filepath.Join(ceo.openyurtDir, enutil.KubeletConfName) +func (c *ConvertEdgeNodeOptions) GetYurthubKubeletConf() string { + return filepath.Join(c.openyurtDir, enutil.KubeletConfName) } -func (ceo *ConvertEdgeNodeOptions)GetKubeletSvcBackup() string { - return fmt.Sprintf(enutil.KubeletSvcBackup, ceo.kubeletService) +func (c *ConvertEdgeNodeOptions) GetKubeletSvcBackup() string { + return fmt.Sprintf(enutil.KubeletSvcBackup, c.kubeletSvcPath) } -func HubHealthcheck(serverHealthzAddr string) error { - intervalTicker := time.NewTicker(checkFrequency) +func HubHealthcheck() error { + serverHealthzUrl, err := url.Parse(fmt.Sprintf("http://%s", enutil.ServerHealthzServer)) + if err != nil { + return err + } + serverHealthzUrl.Path = enutil.ServerHealthzUrlPath + + intervalTicker := time.NewTicker(hubHealthzCheckFrequency) defer intervalTicker.Stop() - retry := 5 + retry := failedRetry for { select { - case <- intervalTicker.C: - _, err := pingClusterHealthz(serverHealthzAddr) + case <-intervalTicker.C: + _, err := healthchecker.PingClusterHealthz(http.DefaultClient, serverHealthzUrl.String()) retry-- if err != nil { if retry > 0 { @@ -336,26 +286,3 @@ func HubHealthcheck(serverHealthzAddr string) error { } } } - -func pingClusterHealthz(addr string) (bool, error) { - resp, err := http.Get(addr) - if err != nil { - return false, err - } - - b, err := ioutil.ReadAll(resp.Body) - defer resp.Body.Close() - if err != nil { - return false, fmt.Errorf("failed to read response of cluster healthz, %v", err) - } - - if resp.StatusCode != http.StatusOK { - return false, fmt.Errorf("response status code is %d", resp.StatusCode) - } - - if strings.ToLower(string(b)) != "ok" { - return false, fmt.Errorf("cluster healthz is %s", string(b)) - } - - return true, nil -} \ No newline at end of file diff --git a/pkg/yurtctl/cmd/revert/edgenode.go b/pkg/yurtctl/cmd/revert/edgenode.go index 578f31498d4..4c2769ba8ab 100644 --- a/pkg/yurtctl/cmd/revert/edgenode.go +++ b/pkg/yurtctl/cmd/revert/edgenode.go @@ -2,26 +2,29 @@ package revert import ( "fmt" - "github.com/alibaba/openyurt/pkg/projectinfo" - "github.com/alibaba/openyurt/pkg/yurtctl/constants" - "github.com/alibaba/openyurt/pkg/yurtctl/lock" - enutil "github.com/alibaba/openyurt/pkg/yurtctl/util/edgenode" - kubeutil "github.com/alibaba/openyurt/pkg/yurtctl/util/kubernetes" + "os" + "path/filepath" + "github.com/spf13/cobra" "github.com/spf13/pflag" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/klog" - "os" - "path/filepath" + nodeutil "k8s.io/kubernetes/pkg/controller/util/node" + + "github.com/alibaba/openyurt/pkg/projectinfo" + "github.com/alibaba/openyurt/pkg/yurtctl/constants" + "github.com/alibaba/openyurt/pkg/yurtctl/lock" + enutil "github.com/alibaba/openyurt/pkg/yurtctl/util/edgenode" + kubeutil "github.com/alibaba/openyurt/pkg/yurtctl/util/kubernetes" ) type RevertEdgeNodeOptions struct { - clientSet *kubernetes.Clientset - - kubeletService string - openyurtDir string - staticPodPath string + clientSet *kubernetes.Clientset + kubeletSvcPath string + openyurtDir string + podMainfestPath string } func NewRevertEdgeNodeOptions() *RevertEdgeNodeOptions { @@ -29,15 +32,15 @@ func NewRevertEdgeNodeOptions() *RevertEdgeNodeOptions { } func NewRevertEdgeNodeCmd() *cobra.Command { - reo := NewRevertEdgeNodeOptions() + r := NewRevertEdgeNodeOptions() cmd := &cobra.Command{ - Use: "edgenode", - Short: "reverts the yurt node to a kubernetes node", + Use: "edgenode", + Short: "reverts the yurt node to a kubernetes node", Run: func(cmd *cobra.Command, _ []string) { - if err := reo.Complete(cmd.Flags()); err != nil { + if err := r.Complete(cmd.Flags()); err != nil { klog.Fatalf("fail to complete the revert edgenode option: %s", err) } - if err := reo.RunRevertEdgeNode(); err != nil { + if err := r.RunRevertEdgeNode(); err != nil { klog.Fatalf("fail to revert the yurt node to a kubernetes node: %s", err) } }, @@ -45,65 +48,69 @@ func NewRevertEdgeNodeCmd() *cobra.Command { return cmd } -func (reo *RevertEdgeNodeOptions) Complete(flags *pflag.FlagSet) (err error) { - reo.clientSet, err = kubeutil.GenClientSet(flags) +func (r *RevertEdgeNodeOptions) Complete(flags *pflag.FlagSet) (err error) { + r.clientSet, err = kubeutil.GenClientSet(flags) if err != nil { return err } - kls := os.Getenv("KUBELET_SVC") - if kls == "" { - kls = enutil.KubeletSvc + kubeletSvcPath := os.Getenv("KUBELET_SVC") + if kubeletSvcPath == "" { + kubeletSvcPath = enutil.KubeletSvcPath } - reo.kubeletService = kls + r.kubeletSvcPath = kubeletSvcPath - oyd := os.Getenv("OPENYURT_DIR") - if oyd == "" { - oyd = enutil.OpenyurtDir + openyurtDir := os.Getenv("OPENYURT_DIR") + if openyurtDir == "" { + openyurtDir = enutil.OpenyurtDir } - reo.openyurtDir = oyd + r.openyurtDir = openyurtDir - spp := os.Getenv("STATIC_POD_PATH") - if spp == "" { - spp = enutil.StaticPodPath + podMainfestPath := os.Getenv("STATIC_POD_PATH") + if podMainfestPath == "" { + podMainfestPath = enutil.StaticPodPath } - reo.staticPodPath = spp + r.podMainfestPath = podMainfestPath return } -func (reo *RevertEdgeNodeOptions) RunRevertEdgeNode() (err error) { - if err = lock.AcquireLock(reo.clientSet); err != nil { +func (r *RevertEdgeNodeOptions) RunRevertEdgeNode() (err error) { + if err = lock.AcquireLock(r.clientSet); err != nil { return } defer func() { - if releaseLockErr := lock.ReleaseLock(reo.clientSet); releaseLockErr != nil { + if releaseLockErr := lock.ReleaseLock(r.clientSet); releaseLockErr != nil { klog.Error(releaseLockErr) } }() klog.V(4).Info("successfully acquire the lock") // 1. check the server version - if err = kubeutil.ValidateServerVersion(reo.clientSet); err != nil { + if err = kubeutil.ValidateServerVersion(r.clientSet); err != nil { return } klog.V(4).Info("the server version is valid") - // 2. remove label from node + // 2. check the state of worker node and remove label from node nodeName, err := enutil.GetNodeName() if err != nil { return err } - node, err := reo.clientSet.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) + node, err := r.clientSet.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) if err != nil { return err } isEdgeNode, ok := node.Labels[projectinfo.GetEdgeWorkerLabelKey()] - node, err = reo.clientSet.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) - if err != nil { - return err - } if ok && isEdgeNode == "true" { + // 2.1. check the state of worker nodes + _, condition := nodeutil.GetNodeCondition(&node.Status, v1.NodeReady) + if condition == nil || condition.Status != v1.ConditionTrue { + klog.Errorf("Cannot do the revert, the status of worker node: %s is not 'Ready'.", node.Name) + return + } + + // 2.2. remove label from node _, foundAutonomy := node.Annotations[constants.AnnotationAutonomy] if foundAutonomy { delete(node.Annotations, constants.AnnotationAutonomy) @@ -112,36 +119,36 @@ func (reo *RevertEdgeNodeOptions) RunRevertEdgeNode() (err error) { if ok { // remove the label for both the cloud node and the edge node delete(node.Labels, projectinfo.GetEdgeWorkerLabelKey()) - if _, err = reo.clientSet.CoreV1().Nodes().Update(node); err != nil { + if _, err = r.clientSet.CoreV1().Nodes().Update(node); err != nil { return } } klog.Info("label alibabacloud.com/is-edge-worker is removed") // 3. remove yurt-hub and revert kubelet service - if err := reo.RevertKubelet(); err != nil { + if err := r.RevertKubelet(); err != nil { return fmt.Errorf("fail to revert kubelet: %v", err) } - if err := reo.RemoveYurthub(); err != nil { + if err := r.RemoveYurthub(); err != nil { return err } return } -func (reo *RevertEdgeNodeOptions) RevertKubelet() error { +func (r *RevertEdgeNodeOptions) RevertKubelet() error { // 1. remove openyurt's kubelet.conf if exist - yurtKubeletConf := reo.GetYurthubKubeletConf() + yurtKubeletConf := r.GetYurthubKubeletConf() if err := os.Remove(yurtKubeletConf); err != nil { return err } - kubeletSvcBk := reo.GetKubeletSvcBackup() + kubeletSvcBk := r.GetKubeletSvcBackup() _, err := enutil.FileExists(kubeletSvcBk) if err != nil { - klog.Errorf("fail to get file %s, will revise the %s directly", kubeletSvcBk, reo.kubeletService) + klog.Errorf("fail to get file %s, will revise the %s directly", kubeletSvcBk, r.kubeletSvcPath) return err } klog.Infof("found backup file %s, will use it to revert the node", kubeletSvcBk) - err = os.Rename(kubeletSvcBk, reo.kubeletService) + err = os.Rename(kubeletSvcBk, r.kubeletSvcPath) if err != nil { return err } @@ -162,9 +169,9 @@ func (reo *RevertEdgeNodeOptions) RevertKubelet() error { return nil } -func (reo *RevertEdgeNodeOptions) RemoveYurthub() error { +func (r *RevertEdgeNodeOptions) RemoveYurthub() error { // 1. remove the yurt-hub.yaml to delete the yurt-hub - yurthubYaml := reo.GetYurthubYaml() + yurthubYaml := r.GetYurthubYaml() err := os.Remove(yurthubYaml) if err != nil { return err @@ -173,14 +180,14 @@ func (reo *RevertEdgeNodeOptions) RemoveYurthub() error { return nil } -func (reo *RevertEdgeNodeOptions) GetYurthubKubeletConf() string { - return filepath.Join(reo.openyurtDir, enutil.KubeletConfName) +func (r *RevertEdgeNodeOptions) GetYurthubKubeletConf() string { + return filepath.Join(r.openyurtDir, enutil.KubeletConfName) } -func (reo *RevertEdgeNodeOptions) GetKubeletSvcBackup() string { - return fmt.Sprintf(enutil.KubeletSvcBackup, reo.kubeletService) +func (r *RevertEdgeNodeOptions) GetKubeletSvcBackup() string { + return fmt.Sprintf(enutil.KubeletSvcBackup, r.kubeletSvcPath) } -func (reo *RevertEdgeNodeOptions) GetYurthubYaml() string { - return filepath.Join(reo.staticPodPath, enutil.YurthubYamlName) -} \ No newline at end of file +func (r *RevertEdgeNodeOptions) GetYurthubYaml() string { + return filepath.Join(r.podMainfestPath, enutil.YurthubYamlName) +} diff --git a/pkg/yurtctl/cmd/revert/revert.go b/pkg/yurtctl/cmd/revert/revert.go index 600ce508d14..40225a928a6 100644 --- a/pkg/yurtctl/cmd/revert/revert.go +++ b/pkg/yurtctl/cmd/revert/revert.go @@ -62,6 +62,8 @@ func NewRevertCmd() *cobra.Command { }, } + cmd.AddCommand(NewRevertEdgeNodeCmd()) + cmd.Flags().String("yurtctl-servant-image", "openyurt/yurtctl-servant:latest", "The yurtctl-servant image.") @@ -222,7 +224,6 @@ func (ro *RevertOptions) RunRevert() (err error) { // 7. remove yurt-hub and revert kubelet service if err = kubeutil.RunServantJobs(ro.clientSet, map[string]string{ - "action": "revert", "yurtctl_servant_image": ro.YurtctlServantImage, }, edgeNodeNames, false); err != nil { diff --git a/pkg/yurtctl/constants/constants.go b/pkg/yurtctl/constants/constants.go index e6cd35d68d0..5b10d942dc1 100644 --- a/pkg/yurtctl/constants/constants.go +++ b/pkg/yurtctl/constants/constants.go @@ -159,47 +159,6 @@ spec: command: - yurt-controller-manager ` - // ServantJobTemplate defines the servant job in yaml format -// ServantJobTemplate = ` -//apiVersion: batch/v1 -//kind: Job -//metadata: -// name: {{.jobName}} -// namespace: kube-system -//spec: -// template: -// spec: -// hostPID: true -// hostNetwork: true -// restartPolicy: OnFailure -// nodeName: {{.nodeName}} -// volumes: -// - name: host-var-tmp -// hostPath: -// path: /var/tmp -// type: Directory -// containers: -// - name: yurtctl-servant -// image: {{.yurtctl_servant_image}} -// imagePullPolicy: Always -// command: -// - /bin/sh -// - -c -// args: -// - "sed -i 's|__yurthub_image__|{{.yurthub_image}}|g;s|__join_token__|{{.joinToken}}|g' /var/lib/openyurt/setup_edgenode && cp /var/lib/openyurt/setup_edgenode /tmp && nsenter -t 1 -m -u -n -i /var/tmp/setup_edgenode {{.action}} " -// securityContext: -// privileged: true -// volumeMounts: -// - mountPath: /tmp -// name: host-var-tmp -// env: -// - name: NODE_NAME -// valueFrom: -// fieldRef: -// fieldPath: spec.nodeName -// - name: STATIC_POD_PATH -// value: {{.pod_manifest_path}} -//` // ConvertServantJobTemplate defines the servant job in yaml format ConvertServantJobTemplate = ` apiVersion: batch/v1 @@ -224,9 +183,10 @@ spec: image: {{.yurtctl_servant_image}} imagePullPolicy: Always command: - - /bin/yurtctl + - /bin/sh + - -c args: - - "convert edgenode --yurthub-image {{.yurthub_image}} --join-token {{.joinToken}} --pod-manifest-path {{.pod_manifest_path}}" + - "nsenter -t 1 -m -u -n -i /bin/yurtctl convert edgenode --yurthub-image {{.yurthub_image}} --join-token {{.joinToken}} --pod-manifest-path {{.pod_manifest_path}}" securityContext: privileged: true volumeMounts: @@ -263,9 +223,10 @@ spec: image: {{.yurtctl_servant_image}} imagePullPolicy: Always command: - - /bin/yurtctl + - /bin/sh + - -c args: - - "revert edgenode" + - "nsenter -t 1 -m -u -n -i /bin/yurtctl revert edgenode" securityContext: privileged: true volumeMounts: diff --git a/pkg/yurtctl/util/edgenode/common.go b/pkg/yurtctl/util/edgenode/common.go index 5fa5a453687..e1fd1468b5e 100644 --- a/pkg/yurtctl/util/edgenode/common.go +++ b/pkg/yurtctl/util/edgenode/common.go @@ -1,20 +1,23 @@ package edgenode const ( - KubeletSvc = "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf" - OpenyurtDir = "/var/lib/openyurt" - StaticPodPath = "/etc/kubernetes/manifests" - YurthubYamlName = "yurt-hub.yaml" - KubeletConfName = "kubelet.conf" + KubeletSvcPath = "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf" + OpenyurtDir = "/var/lib/openyurt" + StaticPodPath = "/etc/kubernetes/manifests" + YurthubYamlName = "yurt-hub.yaml" + KubeletConfName = "kubelet.conf" KubeletSvcBackup = "%s.bk" - Hostname = "/etc/hostname" + Hostname = "/etc/hostname" + KubeletHostname = "--hostname-override=[^\"\\s]*" + KubeletEnvironmentFile = "EnvironmentFile=.*" - DaemonReload = "daemon-reload" - RestartKubeletSvc= "restart kubelet.service" + DaemonReload = "daemon-reload" + RestartKubeletSvc = "restart kubelet.service" - ServerHealthzAddr = "127.0.0.1:10261" - OpenyurtKubeletConf = ` + ServerHealthzServer = "127.0.0.1:10261" + ServerHealthzUrlPath = "/v1/healthz" + OpenyurtKubeletConf = ` apiVersion: v1 clusters: - cluster: diff --git a/pkg/yurtctl/util/edgenode/exec.go b/pkg/yurtctl/util/edgenode/exec.go index 4fe91b2f447..20d915c3c00 100644 --- a/pkg/yurtctl/util/edgenode/exec.go +++ b/pkg/yurtctl/util/edgenode/exec.go @@ -4,8 +4,8 @@ import ( "bytes" "errors" "fmt" - "strings" "os/exec" + "strings" "syscall" ) @@ -26,29 +26,25 @@ func NewCommand(command string) *Command { // Any running error or non-zero exitcode is consider as error func (cmd *Command) Exec() error { var stdoutBuf, stderrBuf bytes.Buffer - cmd.Cmd.Stdout = &stdoutBuf // 标准输出 - cmd.Cmd.Stderr = &stderrBuf // 标准错误 + cmd.Cmd.Stdout = &stdoutBuf + cmd.Cmd.Stderr = &stderrBuf errString := fmt.Sprintf("failed to exec '%s'", cmd.GetCommand()) - // start() & wait() 必须一起使用 - // 资料：https://studygolang.com/articles/3990 - // https://zhuanlan.zhihu.com/p/296409942 - err := cmd.Cmd.Start() // 本质上这个函数开始执行命令，不用等待命令执行结果 + err := cmd.Cmd.Start() if err != nil { errString = fmt.Sprintf("%s, err: %v", errString, err) return errors.New(errString) } - err = cmd.Cmd.Wait() // 等待命令执行结束 + err = cmd.Cmd.Wait() if err != nil { - cmd.StdErr = stderrBuf.Bytes() // 执行错误结果（bytes 形式） + cmd.StdErr = stderrBuf.Bytes() - // 如果 err 类型是 ExitError if exit, ok := err.(*exec.ExitError); ok { - cmd.ExitCode = exit.Sys().(syscall.WaitStatus).ExitStatus() // 获取exit code + cmd.ExitCode = exit.Sys().(syscall.WaitStatus).ExitStatus() errString = fmt.Sprintf("%s, err: %s", errString, stderrBuf.Bytes()) - } else { // 如果 err 类型不是 ExitError，赋值为 1 + } else { cmd.ExitCode = 1 } @@ -57,7 +53,7 @@ func (cmd *Command) Exec() error { return errors.New(errString) } - cmd.StdOut, cmd.StdErr = stdoutBuf.Bytes(), stderrBuf.Bytes() // 命令执行之后的 "标准输出"、"标准错误"（bytes形式） + cmd.StdOut, cmd.StdErr = stdoutBuf.Bytes(), stderrBuf.Bytes() return nil } diff --git a/pkg/yurtctl/util/edgenode/util.go b/pkg/yurtctl/util/edgenode/util.go index 22eb57636a3..ffbd70c3014 100644 --- a/pkg/yurtctl/util/edgenode/util.go +++ b/pkg/yurtctl/util/edgenode/util.go @@ -5,6 +5,7 @@ import ( "io/ioutil" "os" "regexp" + "strings" ) func FileExists(filename string) (bool, error) { @@ -24,7 +25,7 @@ func GetContentFormFile(filename string, regularExpression string) ([]string, er } ct := string(content) reg := regexp.MustCompile(regularExpression) - res := reg.FindAllString(ct,-1) + res := reg.FindAllString(ct, -1) return res, nil } @@ -67,18 +68,38 @@ func CopyFile(sourceFile string, destinationFile string) error { func ReplaceRegularExpression(content string, replace map[string]string) string { for old, new := range replace { reg := regexp.MustCompile(old) - content = reg.ReplaceAllString(content,new) + content = reg.ReplaceAllString(content, new) } - return content } - func GetNodeName() (string, error) { + //1. find --hostname-override in 10-kubeadm.conf + nodeName, err := GetSingleContentFromFile(KubeletSvcPath, KubeletHostname) + if nodeName != "" { + nodeName = strings.Split(nodeName, "=")[1] + return nodeName, nil + } + + //2. find --hostname-override in EnvironmentFile + environmentFiles, err := GetContentFormFile(KubeletSvcPath, KubeletEnvironmentFile) + if err != nil { + return "", err + } + for _, ef := range environmentFiles { + ef = strings.Split(ef, "-")[1] + nodeName, err = GetSingleContentFromFile(ef, KubeletHostname) + if nodeName != "" { + nodeName = strings.Split(nodeName, "=")[1] + return nodeName, nil + } + } + + //3. read nodeName from /etc/hostname content, err := ioutil.ReadFile(Hostname) if err != nil { return "", err } - nodeName := string(content) + nodeName = string(content) return nodeName, nil -} \ No newline at end of file +} diff --git a/pkg/yurtctl/util/kubernetes/util.go b/pkg/yurtctl/util/kubernetes/util.go index f3c0379a25c..3d4fa29c03e 100644 --- a/pkg/yurtctl/util/kubernetes/util.go +++ b/pkg/yurtctl/util/kubernetes/util.go @@ -276,11 +276,11 @@ func RunJobAndCleanup(cliSet *kubernetes.Clientset, job *batchv1.Job, timeout, p // RunServantJobs launchs servant jobs on specified edge nodes func RunServantJobs(cliSet *kubernetes.Clientset, tmplCtx map[string]string, edgeNodeNames []string, convert bool) error { var wg sync.WaitGroup - var sjt string + var servantJobTemplate string if convert { - sjt = constants.ConvertServantJobTemplate + servantJobTemplate = constants.ConvertServantJobTemplate } else { - sjt = constants.RevertServantJobTemplate + servantJobTemplate = constants.RevertServantJobTemplate } for _, nodeName := range edgeNodeNames { action, exist := tmplCtx["action"] @@ -298,7 +298,7 @@ func RunServantJobs(cliSet *kubernetes.Clientset, tmplCtx map[string]string, edg } tmplCtx["nodeName"] = nodeName - jobYaml, err := tmplutil.SubsituteTemplate(sjt, tmplCtx) + jobYaml, err := tmplutil.SubsituteTemplate(servantJobTemplate, tmplCtx) if err != nil { return err } diff --git a/pkg/yurthub/healthchecker/health_checker.go b/pkg/yurthub/healthchecker/health_checker.go index 9f8502dce33..0cc481363ff 100644 --- a/pkg/yurthub/healthchecker/health_checker.go +++ b/pkg/yurthub/healthchecker/health_checker.go @@ -116,7 +116,7 @@ func newChecker(url *url.URL, tp transport.Interface, failedRetry, healthyThresh healthyThreshold: healthyThreshold, } - initHealthyStatus, err := pingClusterHealthz(c.healthzClient, c.serverHealthzAddr) + initHealthyStatus, err := PingClusterHealthz(c.healthzClient, c.serverHealthzAddr) if err != nil { klog.Errorf("cluster(%s) init status: unhealthy, %v", c.serverHealthzAddr, err) } @@ -146,7 +146,7 @@ func (c *checker) healthyCheckLoop(stopCh <-chan struct{}) { return case <-intervalTicker.C: for i := 0; i < c.failedRetry; i++ { - isHealthy, err = pingClusterHealthz(c.healthzClient, c.serverHealthzAddr) + isHealthy, err = PingClusterHealthz(c.healthzClient, c.serverHealthzAddr) if err != nil { klog.V(2).Infof("ping cluster healthz with result, %v", err) if !c.clusterHealthy { @@ -189,7 +189,7 @@ func (c *checker) healthyCheckLoop(stopCh <-chan struct{}) { } } -func pingClusterHealthz(client *http.Client, addr string) (bool, error) { +func PingClusterHealthz(client *http.Client, addr string) (bool, error) { if client == nil { return false, fmt.Errorf("http client is invalid") }