Fixed an issue with detection of private network for autoscaled nodes

src/hetzner-k3s.cr

@@ -8,7 +8,7 @@ require "./cluster/upgrade"
 
 module Hetzner::K3s
   class CLI < Admiral::Command
-    VERSION = "2.0.7"
+    VERSION = "2.0.8"
 
     class Create < Admiral::Command
       define_help description: "create - Create a cluster"

src/hetzner/instance/create.cr

@@ -182,10 +182,10 @@ class Hetzner::Instance::Create
     base_config
   end
 
-  def self.cloud_init(settings, ssh_port = 22, snapshot_os = "default", additional_packages = [] of String, additional_post_create_commands = [] of String, final_commands = [] of String)
+  def self.cloud_init(settings, ssh_port = 22, snapshot_os = "default", additional_packages = [] of String, additional_post_create_commands = [] of String, init_commands = [] of String)
     Crinja.render(CLOUD_INIT_YAML, {
       packages_str: generate_packages_str(snapshot_os, additional_packages),
-      post_create_commands_str: generate_post_create_commands_str(snapshot_os, additional_post_create_commands, final_commands),
+      post_create_commands_str: generate_post_create_commands_str(snapshot_os, additional_post_create_commands, init_commands),
       eth1_str: eth1(snapshot_os),
       growpart_str: growpart(snapshot_os),
       ssh_port: ssh_port
@@ -214,13 +214,12 @@ class Hetzner::Instance::Create
     [
       "hostnamectl set-hostname $(curl http://169.254.169.254/hetzner/v1/metadata/hostname)",
       "update-crypto-policies --set DEFAULT:SHA1 || true",
-      "chmod +x /etc/configure-ssh.sh",
       "/etc/configure-ssh.sh",
       "echo \"nameserver 8.8.8.8\" > /etc/k8s-resolv.conf"
     ]
   end
 
-  def self.generate_post_create_commands_str(snapshot_os, additional_post_create_commands, final_commands)
+  def self.generate_post_create_commands_str(snapshot_os, additional_post_create_commands, init_commands)
     post_create_commands = mandatory_post_create_commands
 
     if snapshot_os == "microos"
@@ -239,7 +238,7 @@ class Hetzner::Instance::Create
       end
     end
 
-    post_create_commands += additional_post_create_commands + final_commands
+    post_create_commands = post_create_commands + init_commands + additional_post_create_commands
 
     "- #{post_create_commands.join("\n- ")}"
   end

src/kubernetes/installer.cr

@@ -177,6 +177,7 @@ class Kubernetes::Installer
       tls_sans: generate_tls_sans(master_count),
       private_network_enabled: settings.networking.private_network.enabled.to_s,
       private_network_test_ip: settings.networking.private_network.subnet.split(".")[0..2].join(".") + ".0",
+      private_network_subnet: settings.networking.private_network.enabled ? settings.networking.private_network.subnet : "",
       cluster_cidr: settings.networking.cluster_cidr,
       service_cidr: settings.networking.service_cidr,
       cluster_dns: settings.networking.cluster_dns,
@@ -194,6 +195,7 @@ class Kubernetes::Installer
       api_server_ip_address: api_server_ip_address,
       private_network_enabled: settings.networking.private_network.enabled.to_s,
       private_network_test_ip: settings.networking.private_network.subnet.split(".")[0..2].join(".") + ".0",
+      private_network_subnet: settings.networking.private_network.enabled ? settings.networking.private_network.subnet : "",
       extra_args: kubelet_args_list
     })
   end

src/kubernetes/software/cluster_autoscaler.cr

@@ -39,13 +39,7 @@ class Kubernetes::Software::ClusterAutoscaler
   end
 
   private def k3s_join_script
-    start_index = worker_install_script.index("touch /etc/initialized") || 0
-    # make sure we early detect, when this line would be changed in the worker install script.
-    # Keeping "cloud init finished"-detection within the autoscaled nodes would deadlock,
-    # since there we run it *during* cloud init.
-    raise "Error: 'touch /etc/initialized' not found in worker_install_script" unless start_index
-    script_part = worker_install_script[start_index..-1]
-    "|\n    #{script_part.gsub("\n", "\n    ")}"
+    "|\n    #{worker_install_script.gsub("\n", "\n    ")}"
   end
 
   private def certificate_path

templates/cloud_init.yaml

@@ -3,13 +3,18 @@ preserve_hostname: true
 
 write_files:
 {{ eth1_str }}
+
 {{ growpart_str }}
-- content: |
+
+- path: /etc/systemd/system/ssh.socket.d/listen.conf
+  content: |
     [Socket]
     ListenStream=
     ListenStream={{ ssh_port}}
-  path: /etc/systemd/system/ssh.socket.d/listen.conf
-- content: |
+
+- path: /etc/configure-ssh.sh
+  permissions: '0755'
+  content: |
     if systemctl is-active ssh.socket > /dev/null 2>&1
     then
       # OpenSSH is using socket activation
@@ -22,7 +27,7 @@ write_files:
       sed -i 's/^#*Port .*/Port {{ ssh_port }}/' /etc/ssh/sshd_config
     fi
     systemctl restart ssh
-  path: /etc/configure-ssh.sh
+
 
 packages: [{{ packages_str }}]
 

templates/master_install_script.sh

@@ -1,28 +1,45 @@
 touch /etc/initialized
 
-if [[ $(< /etc/initialized) != "true" ]]; then
-  systemctl restart NetworkManager || true
-  dhclient eth1 -v || true
-fi
-
 HOSTNAME=$(hostname -f)
 PUBLIC_IP=$(hostname -I | awk '{print $1}')
 
-if [[ "{{ private_network_enabled }}" = "true" ]]; then
+if [ "{{ private_network_enabled }}" = "true" ]; then
+  echo "Using private network " > /var/log/hetzner-k3s.log
+  SUBNET="{{ private_network_subnet }}"
+  SUBNET_PREFIX=$(echo $SUBNET | cut -d'/' -f1 | sed 's/\./\\./g' | sed 's/0$//')
+  MAX_ATTEMPTS=30
+  DELAY=10
+  UP="false"
+
+  for i in $(seq 1 $MAX_ATTEMPTS); do
+    if ip -4 addr show | grep -q "inet $SUBNET_PREFIX"; then
+      echo "Private network IP in subnet $SUBNET is up" 2>&1 | tee -a /var/log/hetzner-k3s.log
+      UP="true"
+      break
+    fi
+    echo "Waiting for private network IP in subnet $SUBNET to be available... (Attempt $i/$MAX_ATTEMPTS)" 2>&1 | tee -a /var/log/hetzner-k3s.log
+    sleep $DELAY
+  done
+
+  if [ "$UP" = "false" ]; then
+    echo "Timeout waiting for private network IP in subnet $SUBNET" 2>&1 | tee -a /var/log/hetzner-k3s.log
+  fi
+
   PRIVATE_IP=$(ip route get {{ private_network_test_ip }} | awk -F"src " 'NR==1{split($2,a," ");print a[1]}')
   NETWORK_INTERFACE=" --flannel-iface=$(ip route get {{ private_network_test_ip }} | awk -F"dev " 'NR==1{split($2,a," ");print a[1]}') "
 else
+  echo "Using public network " > /var/log/hetzner-k3s.log
   PRIVATE_IP="${PUBLIC_IP}"
   NETWORK_INTERFACE=" "
 fi
 
-if [[ "{{ cni }}" = "true" ]] && [[ "{{ cni_mode }}" = "flannel" ]]; then
+if [ "{{ cni }}" = "true" ] && [ "{{ cni_mode }}" = "flannel" ]; then
   FLANNEL_SETTINGS=" {{ flannel_backend }} $NETWORK_INTERFACE "
 else
   FLANNEL_SETTINGS=" {{ flannel_backend }} "
 fi
 
-if [[ "{{ embedded_registry_mirror_enabled }}" = "true" ]]; then
+if [ "{{ embedded_registry_mirror_enabled }}" = "true" ]; then
   EMBEDDED_REGISTRY_MIRROR=" --embedded-registry "
 else
   EMBEDDED_REGISTRY_MIRROR=" "

templates/worker_install_script.sh

@@ -1,17 +1,34 @@
 touch /etc/initialized
 
-if [[ $(< /etc/initialized) != "true" ]]; then
-  systemctl restart NetworkManager || true
-  dhclient eth1 -v || true
-fi
-
 HOSTNAME=$(hostname -f)
 PUBLIC_IP=$(hostname -I | awk '{print $1}')
 
-if [[ "{{ private_network_enabled }}" = "true" ]]; then
+if [ "{{ private_network_enabled }}" = "true" ]; then
+  echo "Using private network " > /var/log/hetzner-k3s.log
+  SUBNET="{{ private_network_subnet }}"
+  SUBNET_PREFIX=$(echo $SUBNET | cut -d'/' -f1 | sed 's/\./\\./g' | sed 's/0$//')
+  MAX_ATTEMPTS=30
+  DELAY=10
+  UP="false"
+
+  for i in $(seq 1 $MAX_ATTEMPTS); do
+    if ip -4 addr show | grep -q "inet $SUBNET_PREFIX"; then
+      echo "Private network IP in subnet $SUBNET is up" 2>&1 | tee -a /var/log/hetzner-k3s.log
+      UP="true"
+      break
+    fi
+    echo "Waiting for private network IP in subnet $SUBNET to be available... (Attempt $i/$MAX_ATTEMPTS)" 2>&1 | tee -a /var/log/hetzner-k3s.log
+    sleep $DELAY
+  done
+
+  if [ "$UP" = "false" ]; then
+    echo "Timeout waiting for private network IP in subnet $SUBNET" 2>&1 | tee -a /var/log/hetzner-k3s.log
+  fi
+
   PRIVATE_IP=$(ip route get {{ private_network_test_ip }} | awk -F"src " 'NR==1{split($2,a," ");print a[1]}')
   NETWORK_INTERFACE=" --flannel-iface=$(ip route get {{ private_network_test_ip }} | awk -F"dev " 'NR==1{split($2,a," ");print a[1]}') "
 else
+  echo "Using public network " > /var/log/hetzner-k3s.log
   PRIVATE_IP="${PUBLIC_IP}"
   NETWORK_INTERFACE=" "
 fi