packet: Wait for cloud DNS records

Ensure DNS records were created at the DNS provider before starting kubelet. This eliminates cases where kubelet attempts to resolve records before they were created. This results in negative DNS caching which in turn breaks the cluster bootstrap process.
kinvolk · May 29, 2020 · 8c5b515 · 8c5b515
1 parent e233f6d
commit 8c5b515
Show file tree

Hide file tree

Showing 7 changed files with 141 additions and 15 deletions.
diff --git a/assets/lokomotive-kubernetes/packet/flatcar-linux/kubernetes/cl/controller.yaml.tmpl b/assets/lokomotive-kubernetes/packet/flatcar-linux/kubernetes/cl/controller.yaml.tmpl
@@ -42,14 +42,13 @@ systemd:
         [Unit]
         Description=Wait for DNS entries
         Wants=systemd-resolved.service
-        Before=kubelet.service
+        Before=kubelet.service etcd-member.service bootkube.service
         [Service]
         Type=oneshot
         RemainAfterExit=true
-        ExecStart=/bin/sh -c 'while ! /usr/bin/grep '^[^#[:space:]]' /etc/resolv.conf > /dev/null; do sleep 1; done'
+        ExecStart=/bin/sh -c 'while ! /usr/bin/grep '^[^#[:space:]]' /etc/resolv.conf > /dev/null; do sleep 1; done; /opt/wait-for-dns ${dns_zone} ${cluster_name}-private 3600'
         [Install]
-        RequiredBy=kubelet.service
-        RequiredBy=etcd-member.service
+        RequiredBy=kubelet.service etcd-member.service bootkube.service
     - name: create-etcd-config.service
       # This service will extract value of private interface from the env var file `/run/metadata/flatcar`.
       # And then assign it to the variables that which will be stored in file `/etc/kubernetes/etcd.config`,
@@ -323,6 +322,64 @@ storage:
           kind: KubeletConfiguration
           cgroupDriver: "$${docker_cgroup_driver}"
           EOF
+    - path: /opt/wait-for-dns
+      filesystem: root
+      mode: 0544
+      contents:
+        inline: |
+          #!/bin/bash
+          # TODO: Workaround for https://github.com/flatcar-linux/Flatcar/issues/123.
+          function dig {
+              docker run -i --rm quay.io/kinvolk/alpine-dig:3.9.6 dig "$@" 2>/dev/null
+          }
+          if [[ $# -ne 3 ]]; then
+              echo "Usage: $0 <zone> <record> <max_attempts>"
+              exit 1
+          fi
+          zone=$1
+          record=$2
+          max_attempts=$3
+          echo "Figuring out the nameservers for $zone"
+          nameservers=""
+          counter=0
+          while [[ $counter -lt $max_attempts ]]; do
+              out=$(dig +short +timeout=2 "$zone" ns)
+              ret=$?
+              if [[ $ret -eq 0 && "$out" != "" ]]; then
+                  nameservers=$out
+                  break
+              fi
+              echo "Failed with exit code $ret: $out"
+              sleep 1
+              counter=$((counter+1))
+          done
+          if [[ "$nameservers" == "" ]]; then
+              echo "Could not resolve nameservers for $zone"
+              exit 1
+          fi
+          for ns in $nameservers; do
+              echo "Polling $ns for $record.$zone..."
+              counter=0
+              ok=false
+              while [[ $counter -lt $max_attempts ]]; do
+                  out=$(dig +short +timeout=2 @"$ns" "$record"."$zone" a)
+                  ret=$?
+                  if [[ $ret -eq 0 && "$out" != "" ]]; then
+                      echo "Looks good!"
+                      ok=true
+                      break
+                  fi
+                  echo "Not available yet"
+                  sleep 1
+                  counter=$((counter+1))
+              done
+              if ! $ok; then
+                  echo "$record.$zone didn't become available within the allowed time"
+                  exit 1
+              fi
+          done
+          echo "$record.$zone is available on all nameservers"
+          exit 0
 passwd:
   users:
   - name: core

diff --git a/assets/lokomotive-kubernetes/packet/flatcar-linux/kubernetes/controllers.tf b/assets/lokomotive-kubernetes/packet/flatcar-linux/kubernetes/controllers.tf
@@ -53,6 +53,8 @@ data "ct_config" "controller-ignitions" {
     k8s_dns_service_ip    = cidrhost(var.service_cidr, 10)
     cluster_domain_suffix = var.cluster_domain_suffix
     controller_count      = var.controller_count
+    dns_zone              = var.dns_zone
+    cluster_name          = var.cluster_name
 
     # we need to prepend a prefix 'docker://' for arm64, because arm64 images
     # on quay prevent us from downloading ACI correctly.

diff --git a/assets/lokomotive-kubernetes/packet/flatcar-linux/kubernetes/workers/cl/worker.yaml.tmpl b/assets/lokomotive-kubernetes/packet/flatcar-linux/kubernetes/workers/cl/worker.yaml.tmpl
@@ -31,7 +31,7 @@ systemd:
         [Service]
         Type=oneshot
         RemainAfterExit=true
-        ExecStart=/bin/sh -c 'while ! /usr/bin/grep '^[^#[:space:]]' /etc/resolv.conf > /dev/null; do sleep 1; done'
+        ExecStart=/bin/sh -c 'while ! /usr/bin/grep '^[^#[:space:]]' /etc/resolv.conf > /dev/null; do sleep 1; done; /opt/wait-for-dns ${dns_zone} ${cluster_name}-private 3600'
         [Install]
         RequiredBy=kubelet.service
     - name: coreos-metadata.service
@@ -351,6 +351,64 @@ storage:
           kind: KubeletConfiguration
           cgroupDriver: "$${docker_cgroup_driver}"
           EOF
+    - path: /opt/wait-for-dns
+      filesystem: root
+      mode: 0544
+      contents:
+        inline: |
+          #!/bin/bash
+          # TODO: Workaround for https://github.com/flatcar-linux/Flatcar/issues/123.
+          function dig {
+              docker run -i --rm quay.io/kinvolk/alpine-dig:3.9.6 dig "$@" 2>/dev/null
+          }
+          if [[ $# -ne 3 ]]; then
+              echo "Usage: $0 <zone> <record> <max_attempts>"
+              exit 1
+          fi
+          zone=$1
+          record=$2
+          max_attempts=$3
+          echo "Figuring out the nameservers for $zone"
+          nameservers=""
+          counter=0
+          while [[ $counter -lt $max_attempts ]]; do
+              out=$(dig +short +timeout=2 "$zone" ns)
+              ret=$?
+              if [[ $ret -eq 0 && "$out" != "" ]]; then
+                  nameservers=$out
+                  break
+              fi
+              echo "Failed with exit code $ret: $out"
+              sleep 1
+              counter=$((counter+1))
+          done
+          if [[ "$nameservers" == "" ]]; then
+              echo "Could not resolve nameservers for $zone"
+              exit 1
+          fi
+          for ns in $nameservers; do
+              echo "Polling $ns for $record.$zone..."
+              counter=0
+              ok=false
+              while [[ $counter -lt $max_attempts ]]; do
+                  out=$(dig +short +timeout=2 @"$ns" "$record"."$zone" a)
+                  ret=$?
+                  if [[ $ret -eq 0 && "$out" != "" ]]; then
+                      echo "Looks good!"
+                      ok=true
+                      break
+                  fi
+                  echo "Not available yet"
+                  sleep 1
+                  counter=$((counter+1))
+              done
+              if ! $ok; then
+                  echo "$record.$zone didn't become available within the allowed time"
+                  exit 1
+              fi
+          done
+          echo "$record.$zone is available on all nameservers"
+          exit 0
 passwd:
   users:
   - name: core

diff --git a/assets/lokomotive-kubernetes/packet/flatcar-linux/kubernetes/workers/variables.tf b/assets/lokomotive-kubernetes/packet/flatcar-linux/kubernetes/workers/variables.tf
@@ -175,3 +175,8 @@ variable "nodes_depend_on" {
   type        = list(any)
   default     = null
 }
+
+variable "dns_zone" {
+  type        = string
+  description = "DNS Zone (e.g. example.com)"
+}
diff --git a/assets/lokomotive-kubernetes/packet/flatcar-linux/kubernetes/workers/workers.tf b/assets/lokomotive-kubernetes/packet/flatcar-linux/kubernetes/workers/workers.tf
@@ -65,6 +65,8 @@ data "ct_config" "ignitions" {
       setup_raid_hdd        = var.setup_raid_hdd
       setup_raid_ssd        = var.setup_raid_ssd
       setup_raid_ssd_fs     = var.setup_raid_ssd_fs
+      cluster_name          = var.cluster_name
+      dns_zone              = var.dns_zone
     }
   )
   platform = "packet"

diff --git a/pkg/assets/generated_assets.go b/pkg/assets/generated_assets.go
diff --git a/pkg/platform/packet/template.go b/pkg/platform/packet/template.go
@@ -121,6 +121,8 @@ module "worker-{{ $pool.Name }}" {
     packet   = packet.default
   }
 
+  dns_zone = "{{$.Config.DNS.Zone}}"
+
   ssh_keys  = {{$.SSHPublicKeys}}
 
   cluster_name = "{{$.Config.ClusterName}}"