VCDA-2776: Merge code of TKGm with immutable infra changes. (vmware#1179

) - also implement workaround for gosc issue - also implement any missing functionality Signed-off-by: Arun M. Krishnakumar <akrishnakuma@vmware.com>
andrew-ni · Sep 4, 2021 · f2cf811 · f2cf811
1 parent caabbc0
commit f2cf811
Show file tree

Hide file tree

Showing 10 changed files with 888 additions and 1,158 deletions.
diff --git a/.gitignore b/.gitignore
@@ -101,3 +101,6 @@ _site/
 cse_cache
 
 tests/private.config.yml
+
+# intellij metadata
+.idea/*
diff --git a/cluster_scripts/v2_x_tkgm/control_plane.sh b/cluster_scripts/v2_x_tkgm/control_plane.sh
@@ -1,37 +1,102 @@
 #!/usr/bin/env bash
+
+catch() {{
+   vmtoolsd --cmd "info-set guestinfo.post_customization_script_execution_status $?"
+   error_message="$(date) $(caller): $BASH_COMMAND"
+   echo "$error_message" &>> /var/log/cse/customization/error.log
+   vmtoolsd --cmd "info-set guestinfo.post_customization_script_execution_failure_reason $error_message"
+}}
+
+mkdir -p /var/log/cse/customization
+
+trap 'catch $? $LINENO' ERR
+
 set -e
 
+echo "$(date) This script was called with $1" &>> /var/log/cse/customization/status.log
+
+if [ "$1" == "precustomization" ]
+then
+  echo "$(date) Exiting early since phase is [$1]" &>> /var/log/cse/customization/status.log
+  exit 0
+elif [ "$1" != "postcustomization" ]
+then
+  echo "$(date) Exiting early since phase is [$1]" &>> /var/log/cse/customization/status.log
+  exit 0
+fi
+
+echo "$(date) Post Customization script execution in progress" &>> /var/log/cse/customization/status.log
+
 kubeadm_config_path=/root/kubeadm-defaults.conf
-vcloud_basic_auth_path=/root/vcloud-basic-auth.yaml
 vcloud_configmap_path=/root/vcloud-configmap.yaml
 vcloud_ccm_path=/root/cloud-director-ccm.yaml
 csi_driver_path=/root/csi-driver.yaml
 csi_controller_path=/root/csi-controller.yaml
 csi_node_path=/root/csi-node.yaml
 
-# tag images
-coredns_image_version=""
-etcd_image_version=""
-kubernetes_version=""
-for image in "coredns" "etcd" "kube-proxy" "kube-apiserver" "kube-controller-manager" "kube-scheduler"
-do
-  image_ref=$(ctr -n=k8s.io image list | cut -d" " -f1 | grep $image)
-  ref_path=$(echo $image_ref | sed 's/:.*//')
-  new_tag_version=$(echo $image_ref | sed 's/.*://' | sed 's/_/-/')
-  ctr -n=k8s.io image tag $image_ref $ref_path:$new_tag_version
-
-  # save image tags for later
-  if [[ "$image" = "coredns" ]]; then
-    coredns_image_version=$new_tag_version
-  elif [[ "$image" = "etcd" ]]; then
-    etcd_image_version=$new_tag_version
-  elif [[ "$image" = "kube-proxy" ]]; then # selecting other kube-* images would work too
-    kubernetes_version=$new_tag_version
+
+# This is a simple command but its execution is crucial to kubeadm join. There are a few versions of ubuntu
+# where the dbus.service is not started in a timely enough manner to set the hostname correctly. Hence
+# this needs to be set by us.
+vmtoolsd --cmd "info-set guestinfo.postcustomization.hostname.status in_progress"
+  hostnamectl set-hostname {vm_host_name}
+vmtoolsd --cmd "info-set guestinfo.postcustomization.hostname.status successful"
+
+
+vmtoolsd --cmd "info-set guestinfo.postcustomization.networkconfiguration.status in_progress"
+  echo 'net.ipv6.conf.all.disable_ipv6 = 1' >> /etc/sysctl.conf
+  echo 'net.ipv6.conf.default.disable_ipv6 = 1' >> /etc/sysctl.conf
+  echo 'net.ipv6.conf.lo.disable_ipv6 = 1' >> /etc/sysctl.conf
+  sudo sysctl -p
+
+  # also remove ipv6 localhost entry from /etc/hosts
+  sed -i 's/::1/127.0.0.1/g' /etc/hosts || true
+vmtoolsd --cmd "info-set guestinfo.postcustomization.networkconfiguration.status successful"
+
+
+vmtoolsd --cmd "info-set guestinfo.postcustomization.store.sshkey.status in_progress"
+  ssh_key="{ssh_key}"
+  if [[ ! -z "$ssh_key" ]];
+  then
+    mkdir -p /root/.ssh
+    echo $ssh_key >> /root/.ssh/authorized_keys
+    chmod -R go-rwx /root/.ssh
   fi
-done
+vmtoolsd --cmd "info-set guestinfo.postcustomization.store.sshkey.status successful"
+
+
+vmtoolsd --cmd "info-set guestinfo.postcustomization.nameserverconfig.resolvconf.status in_progress"
+  echo 'nameserver 8.8.8.8
+nameserver 8.8.4.4
+nameserver 10.16.188.210
+nameserver 10.118.254.1' > /etc/resolv.conf
+vmtoolsd --cmd "info-set guestinfo.postcustomization.nameserverconfig.resolvconf.status successful"
 
-# create /root/kubeadm-defaults.conf
-echo "---
+
+vmtoolsd --cmd "info-set guestinfo.postcustomization.kubeinit.status in_progress"
+  # tag images
+  coredns_image_version=""
+  etcd_image_version=""
+  kubernetes_version=""
+  for image in "coredns" "etcd" "kube-proxy" "kube-apiserver" "kube-controller-manager" "kube-scheduler"
+  do
+    image_ref=$(ctr -n=k8s.io image list | cut -d" " -f1 | grep $image)
+    ref_path=$(echo $image_ref | sed 's/:.*//')
+    new_tag_version=$(echo $image_ref | sed 's/.*://' | sed 's/_/-/')
+    ctr -n=k8s.io image tag $image_ref $ref_path:$new_tag_version
+
+    # save image tags for later
+    if [[ "$image" = "coredns" ]]; then
+      coredns_image_version=$new_tag_version
+    elif [[ "$image" = "etcd" ]]; then
+      etcd_image_version=$new_tag_version
+    elif [[ "$image" = "kube-proxy" ]]; then # selecting other kube-* images would work too
+      kubernetes_version=$new_tag_version
+    fi
+  done
+
+  # create /root/kubeadm-defaults.conf
+  echo "---
 apiVersion: kubeadm.k8s.io/v1beta2
 kind: InitConfiguration
 bootstrapTokens:
@@ -48,6 +113,7 @@ nodeRegistration:
 ---
 apiVersion: kubeadm.k8s.io/v1beta2
 kind: ClusterConfiguration
+controlPlaneEndpoint: "{control_plane_endpoint}"
 dns:
   type: CoreDNS
   imageRepository: projects.registry.vmware.com/tkg
@@ -62,35 +128,43 @@ networking:
 imageRepository: projects.registry.vmware.com/tkg
 kubernetesVersion: $kubernetes_version
 ---" > /root/kubeadm-defaults.conf
-kubeadm init --config $kubeadm_config_path > /root/kubeadm-init.out
-
-mkdir -p /root/.kube
-cp -f /etc/kubernetes/admin.conf /root/.kube/config
-chown $(id -u):$(id -g) /root/.kube/config
-
-kubectl apply -f https://github.com/vmware-tanzu/antrea/releases/download/v0.11.3/antrea.yml
-systemctl restart kubelet
-while [[ `systemctl is-active kubelet` != 'active' ]]; do echo 'waiting for kubelet'; sleep 5; done
-
-# Download cpi and csi yaml
-#wget -O /root/vcloud-basic-auth.yaml https://raw.githubusercontent.com/vmware/cloud-provider-for-cloud-director/main/manifests/vcloud-basic-auth.yaml
-#wget -O /root/vcloud-configmap.yaml https://raw.githubusercontent.com/vmware/cloud-provider-for-cloud-director/main/manifests/vcloud-configmap.yaml
-#wget -O /root/cloud-director-ccm.yaml https://raw.githubusercontent.com/vmware/cloud-provider-for-cloud-director/main/manifests/cloud-director-ccm.yaml
-# TODO: change to use main branch links
-wget -O $vcloud_basic_auth_path https://raw.githubusercontent.com/ltimothy7/cloud-provider-for-cloud-director/auth_mount_internal/manifests/vcloud-basic-auth.yaml
-wget -O $vcloud_configmap_path https://raw.githubusercontent.com/ltimothy7/cloud-provider-for-cloud-director/auth_mount_internal/manifests/vcloud-configmap.yaml
-wget -O $vcloud_ccm_path https://raw.githubusercontent.com/ltimothy7/cloud-provider-for-cloud-director/auth_mount_internal/manifests/cloud-director-ccm.yaml
-wget -O $csi_driver_path https://github.com/vmware/cloud-director-named-disk-csi-driver/raw/main/manifests/csi-driver.yaml
-wget -O $csi_controller_path https://github.com/vmware/cloud-director-named-disk-csi-driver/raw/main/manifests/csi-controller.yaml
-wget -O $csi_node_path https://github.com/vmware/cloud-director-named-disk-csi-driver/raw/main/manifests/csi-node.yaml
-
-# TODO: look into if not https vcd host
-sed -i 's/BASE64_USERNAME/{base64_username}/; s/BASE64_PASSWORD/{base64_password}/' $vcloud_basic_auth_path
-sed -i 's/VCD_HOST/"https:\/\/{vcd_host}"/; s/ORG/"{org}"/; s/OVDC/"{ovdc}"/; s/NETWORK/"{ovdc_network}"/; s/VIP_SUBNET_CIDR/"{vip_subnet_cidr_ip}\/{vip_subnet_cidr_suffix}"/; s/CLUSTER_ID/"{cluster_id}"/' $vcloud_configmap_path
-
-kubectl apply -f $vcloud_basic_auth_path
-kubectl apply -f $vcloud_configmap_path
-kubectl apply -f $vcloud_ccm_path
-kubectl apply -f $csi_driver_path
-kubectl apply -f $csi_controller_path
-kubectl apply -f $csi_node_path
+  kubeadm init --config $kubeadm_config_path > /root/kubeadm-init.out
+  export KUBECONFIG=/etc/kubernetes/admin.conf
+vmtoolsd --cmd "info-set guestinfo.kubeconfig $(cat /etc/kubernetes/admin.conf | base64 | tr -d '\n')"
+vmtoolsd --cmd "info-set guestinfo.postcustomization.kubeinit.status successful"
+
+
+vmtoolsd --cmd "info-set guestinfo.postcustomization.kubectl.apply.cni.status in_progress"
+  kubectl apply -f https://github.com/vmware-tanzu/antrea/releases/download/{antrea_cni_version}/antrea.yml
+vmtoolsd --cmd "info-set guestinfo.postcustomization.kubectl.apply.cni.status successful"
+
+
+vmtoolsd --cmd "info-set guestinfo.postcustomization.kubectl.cpi.install.status in_progress"
+  # TODO: change to use main branch links
+  wget -O $vcloud_configmap_path https://raw.githubusercontent.com/ltimothy7/cloud-provider-for-cloud-director/auth_mount_internal/manifests/vcloud-configmap.yaml
+  wget -O $vcloud_ccm_path https://raw.githubusercontent.com/ltimothy7/cloud-provider-for-cloud-director/auth_mount_internal/manifests/cloud-director-ccm.yaml
+
+  kubectl apply -f $vcloud_configmap_path
+  kubectl apply -f $vcloud_ccm_path
+vmtoolsd --cmd "info-set guestinfo.postcustomization.kubectl.cpi.install.status successful"
+
+
+vmtoolsd --cmd "info-set guestinfo.postcustomization.kubectl.csi.install.status in_progress"
+  wget -O $csi_driver_path https://github.com/vmware/cloud-director-named-disk-csi-driver/raw/main/manifests/csi-driver.yaml
+  wget -O $csi_controller_path https://github.com/vmware/cloud-director-named-disk-csi-driver/raw/main/manifests/csi-controller.yaml
+  wget -O $csi_node_path https://github.com/vmware/cloud-director-named-disk-csi-driver/raw/main/manifests/csi-node.yaml
+
+  kubectl apply -f $csi_driver_path
+  kubectl apply -f $csi_controller_path
+  kubectl apply -f $csi_node_path
+vmtoolsd --cmd "info-set guestinfo.postcustomization.kubectl.csi.install.status successful"
+
+
+vmtoolsd --cmd "info-set guestinfo.postcustomization.kubeadm.token.generate.status in_progress"
+  kubeadm_join_info=$(kubeadm token create --print-join-command 2> /dev/null)
+  vmtoolsd --cmd "info-set guestinfo.postcustomization.kubeadm.token.info $kubeadm_join_info"
+vmtoolsd --cmd "info-set guestinfo.postcustomization.kubeadm.token.generate.status successful"
+
+
+echo "$(date) post customization script execution completed" &>> /var/log/cse/customization/status.log
+exit 0
diff --git a/cluster_scripts/v2_x_tkgm/node.sh b/cluster_scripts/v2_x_tkgm/node.sh
@@ -1,9 +1,96 @@
 #!/usr/bin/env bash
+
+catch() {{
+   vmtoolsd --cmd "info-set guestinfo.post_customization_script_execution_status $?"
+   error_message="$(date) $(caller): $BASH_COMMAND"
+   echo "$error_message" &>> /var/log/cse/customization/error.log
+   vmtoolsd --cmd "info-set guestinfo.post_customization_script_execution_failure_reason $error_message"
+}}
+
+mkdir -p /var/log/cse/customization
+
+trap 'catch $? $LINENO' ERR
+
 set -e
 
-kubeadm_config_path=/root/kubeadm-defaults-join.conf
+echo "$(date) This script was called with $1" &>> /var/log/cse/customization/status.log
+
+if [ "$1" == "precustomization" ]
+then
+  echo "$(date) Exiting early since phase is [$1]" &>> /var/log/cse/customization/status.log
+  vmtoolsd --cmd "info-set guestinfo.precustomization.script.status successful"
+  exit 0
+elif [ "$1" != "postcustomization" ]
+then
+  echo "$(date) Exiting early since phase is [$1]" &>> /var/log/cse/customization/status.log
+  exit 0
+fi
+
+echo "$(date) Post Customization script execution in progress" &>> /var/log/cse/customization/status.log
+
+# This is a simple command but its execution is crucial to kubeadm join. There are a few versions of ubuntu
+# where the dbus.service is not started in a timely enough manner to set the hostname correctly. Hence
+# this needs to be set by us.
+vmtoolsd --cmd "info-set guestinfo.postcustomization.hostname.status in_progress"
+hostnamectl set-hostname {vm_host_name}
+vmtoolsd --cmd "info-set guestinfo.postcustomization.hostname.status successful"
+
+
+vmtoolsd --cmd "info-set guestinfo.postcustomization.networkconfiguration.status in_progress"
+  echo 'net.ipv6.conf.all.disable_ipv6 = 1' >> /etc/sysctl.conf
+  echo 'net.ipv6.conf.default.disable_ipv6 = 1' >> /etc/sysctl.conf
+  echo 'net.ipv6.conf.lo.disable_ipv6 = 1' >> /etc/sysctl.conf
+  sudo sysctl -p
+
+  # also remove ipv6 localhost entry from /etc/hosts
+  sed -i 's/::1/127.0.0.1/g' /etc/hosts || true
+vmtoolsd --cmd "info-set guestinfo.postcustomization.networkconfiguration.status successful"
+
 
-echo "---
+vmtoolsd --cmd "info-set guestinfo.postcustomization.store.sshkey.status in_progress"
+  ssh_key="{ssh_key}"
+  if [[ ! -z "$ssh_key" ]];
+  then
+    mkdir -p /root/.ssh
+    echo $ssh_key >> /root/.ssh/authorized_keys
+    chmod -R go-rwx /root/.ssh
+  fi
+vmtoolsd --cmd "info-set guestinfo.postcustomization.store.sshkey.status successful"
+
+
+vmtoolsd --cmd "info-set guestinfo.postcustomization.nameserverconfig.resolvconf.status in_progress"
+  echo 'nameserver 8.8.8.8
+nameserver 8.8.4.4
+nameserver 10.16.188.210
+nameserver 10.118.254.1' > /etc/resolv.conf
+vmtoolsd --cmd "info-set guestinfo.postcustomization.nameserverconfig.resolvconf.status successful"
+
+
+vmtoolsd --cmd "info-set guestinfo.postcustomization.kubeadm.node.join.status in_progress"
+  kubeadm_config_path=/root/kubeadm-defaults-join.conf
+
+  # tag images
+  coredns_image_version=""
+  etcd_image_version=""
+  kubernetes_version=""
+  for image in "coredns" "etcd" "kube-proxy" "kube-apiserver" "kube-controller-manager" "kube-scheduler"
+  do
+    image_ref=$(ctr -n=k8s.io image list | cut -d" " -f1 | grep $image)
+    ref_path=$(echo $image_ref | sed 's/:.*//')
+    new_tag_version=$(echo $image_ref | sed 's/.*://' | sed 's/_/-/')
+    ctr -n=k8s.io image tag $image_ref $ref_path:$new_tag_version
+
+    # save image tags for later
+    if [[ "$image" = "coredns" ]]; then
+      coredns_image_version=$new_tag_version
+    elif [[ "$image" = "etcd" ]]; then
+      etcd_image_version=$new_tag_version
+    elif [[ "$image" = "kube-proxy" ]]; then # selecting other kube-* images would work too
+      kubernetes_version=$new_tag_version
+    fi
+  done
+
+  echo "---
 apiVersion: kubeadm.k8s.io/v1beta2
 kind: JoinConfiguration
 caCertPath: /etc/kubernetes/pki/ca.crt
@@ -18,6 +105,12 @@ nodeRegistration:
   criSocket: /run/containerd/containerd.sock
   kubeletExtraArgs:
     cloud-provider: external
-" > /root/kubeadm-defaults-join.conf
+---" > /root/kubeadm-defaults-join.conf
+
+  kubeadm join --config $kubeadm_config_path
+vmtoolsd --cmd "info-set guestinfo.postcustomization.kubeadm.node.join.status successful"
+
+echo "$(date) post customization script execution completed" &>> /var/log/cse/customization/status.log
+
+exit 0
 
-kubeadm join --config $kubeadm_config_path
diff --git a/container_service_extension/common/constants/server_constants.py b/container_service_extension/common/constants/server_constants.py
@@ -768,10 +768,25 @@ class PostCustomizationStatus(Enum):
     SUCCESSFUL = 'successful'
 
 
+@unique
+class ToolsDeployPkgCustomizationStatus(Enum):
+    NONE = None
+    IN_PROGRESS = 'Started'
+    SUCCESSFUL = 'Successful'
+
+
+@unique
+class PreCustomizationPhase(Enum):
+    POST_BOOT_CUSTOMIZATION_SERVICE_SETUP = 'guestinfo.gc.status'
+
+
 @unique
 class PostCustomizationPhase(Enum):
+    HOSTNAME_SETUP = 'guestinfo.postcustomization.hostname.status'
+    NETWORK_CONFIGURATION = 'guestinfo.postcustomization.networkconfiguration.status'
     STORE_SSH_KEY = 'guestinfo.postcustomization.store.sshkey.status'
     KUBEADM_INIT = 'guestinfo.postcustomization.kubeinit.status'
+    NAMESERVER_SETUP = 'guestinfo.postcustomization.nameserverconfig.resolvconf.status'
     KUBECTL_APPLY_CNI = 'guestinfo.postcustomization.kubectl.apply.cni.status'  # noqa: E501
     KUBEADM_TOKEN_GENERATE = 'guestinfo.postcustomization.kubeadm.token.generate.status'  # noqa: E501
     KUBEADM_NODE_JOIN = 'guestinfo.postcustomization.kubeadm.node.join.status'

diff --git a/container_service_extension/common/constants/shared_constants.py b/container_service_extension/common/constants/shared_constants.py
@@ -40,7 +40,6 @@ class ClusterEntityKind(Enum):
                        ClusterEntityKind.TKG_PLUS.value,
                        ClusterEntityKind.TKG_M.value]
 
-
 # Cluster runtimes and placement policies
 NATIVE_CLUSTER_RUNTIME_INTERNAL_NAME = 'native'
 TKG_PLUS_CLUSTER_RUNTIME_INTERNAL_NAME = 'tkgplus'
@@ -60,7 +59,7 @@ class ClusterEntityKind(Enum):
 RUNTIME_INTERNAL_NAME_TO_DISPLAY_NAME_MAP = {
     NATIVE_CLUSTER_RUNTIME_INTERNAL_NAME: ClusterEntityKind.NATIVE.value,
     TKG_PLUS_CLUSTER_RUNTIME_INTERNAL_NAME: ClusterEntityKind.TKG_PLUS.value,
-    TKG_M_CLUSTER_RUNTIME_INTERNAL_NAME: ClusterEntityKind.TKG_M.value
+    TKG_M_CLUSTER_RUNTIME_INTERNAL_NAME: ClusterEntityKind.TKG_M.value,
 }
 
 # CSE Server Busy string