Fixes various issues in vSphere Terraform code (kubernetes-sigs#8178)

* Fixes various issues in vSphere Terraform code Provided to address various shortcomings and to fix the following issue in upstream Kubespray: kubernetes-sigs#8176 * Resolves Terraform formatting issues * Sets default prefix to human-readable name * Documents new default prefix in README
southbridgeio · Jun 28, 2023 · 47f9bf5 · 47f9bf5
1 parent 88023ac
commit 47f9bf5
Show file tree

Hide file tree

Showing 9 changed files with 87 additions and 53 deletions.
diff --git a/contrib/terraform/vsphere/README.md b/contrib/terraform/vsphere/README.md
@@ -1,6 +1,6 @@
-# Kubernetes on Exoscale with Terraform
+# Kubernetes on vSphere with Terraform
 
-Provision a Kubernetes cluster on [vSphere](https://www.vmware.com/se/products/vsphere.html) using Terraform and Kubespray.
+Provision a Kubernetes cluster on [vSphere](https://www.vmware.com/products/vsphere.html) using Terraform and Kubespray.
 
 ## Overview
 
@@ -98,20 +98,32 @@ ansible-playbook -i inventory.ini ../../cluster.yml -b -v
 
 * `machines`: Machines to provision. Key of this object will be used as the name of the machine
   * `node_type`: The role of this node *(master|worker)*
-  * `ip`: The IP address with the netmask (CIDR notation)
+  * `ip`: The IP address of the machine
+  * `netmask`: The netmask to use (to be used on the right hand side in CIDR notation, e.g., `24`)
+* `network`: The name of the network to attach the machines to
 * `gateway`: The IP address of the network gateway
-* `ssh_public_keys`: List of public SSH keys to install on all machines
 * `vsphere_datacenter`: The identifier of vSphere data center
 * `vsphere_compute_cluster`: The identifier of vSphere compute cluster
 * `vsphere_datastore`: The identifier of vSphere data store
 * `vsphere_server`: The address of vSphere server
 * `vsphere_hostname`: The IP address of vSphere hostname
-* `template_name`: The name of a base image (the image has to be uploaded to vSphere beforehand)
+* `ssh_public_keys`: List of public SSH keys to install on all machines
+* `template_name`: The name of a base image (the OVF template be defined in vSphere beforehand)
 
 ### Optional
 
-* `prefix`: Prefix to use for all resources, required to be unique for all clusters in the same project *(Defaults to `default`)*
-* `dns_primary`: The IP address of primary DNS server *(Defaults to `8.8.4.4`)*
-* `dns_secondary`:The IP address of secondary DNS server *(Defaults to `8.8.8.8`)*
+* `folder`: Name of the folder to put all machines in (default: `""`)
+* `prefix`: Prefix to use for all resources, required to be unique for all clusters in the same project (default: `"k8s"`)
+* `inventory_file`: Name of the generated inventory file for Kubespray to use in the Ansible step (default: `inventory.ini`)
+* `dns_primary`: The IP address of primary DNS server (default: `8.8.4.4`)
+* `dns_secondary`: The IP address of secondary DNS server (default: `8.8.8.8`)
+* `firmware`: Firmware to use (default: `bios`)
+* `hardware_version`: The version of the hardware (default: `15`)
+* `master_cores`: The number of CPU cores for the master nodes (default: 4)
+* `master_memory`: The amount of RAM for the master nodes in MB (default: 4096)
+* `master_disk_size`: The amount of disk space for the master nodes in GB (default: 20)
+* `worker_cores`: The number of CPU cores for the worker nodes (default: 16)
+* `worker_memory`: The amount of RAM for the worker nodes in MB (default: 8192)
+* `worker_disk_size`: The amount of disk space for the worker nodes in GB (default: 100)
 
 An example variables file can be found `default.tfvars`
diff --git a/contrib/terraform/vsphere/default.tfvars b/contrib/terraform/vsphere/default.tfvars
@@ -1,23 +1,28 @@
-prefix = "default"
+prefix = "k8s"
 
 inventory_file = "inventory.ini"
 
+network = "VM Network"
+
 machines = {
   "master-0" : {
     "node_type" : "master",
-    "ip" : "i-did-not-read-the-docs" # e.g. 192.168.0.2/24
+    "ip" : "i-did-not-read-the-docs", # e.g. 192.168.0.10
+    "netmask" : "24"
   },
   "worker-0" : {
     "node_type" : "worker",
-    "ip" : "i-did-not-read-the-docs" # e.g. 192.168.0.2/24
+    "ip" : "i-did-not-read-the-docs", # e.g. 192.168.0.20
+    "netmask" : "24"
   },
   "worker-1" : {
     "node_type" : "worker",
-    "ip" : "i-did-not-read-the-docs" # e.g. 192.168.0.2/24
+    "ip" : "i-did-not-read-the-docs", # e.g. 192.168.0.21
+    "netmask" : "24"
   }
 }
 
-gateway = "i-did-not-read-the-docs" # e.g. 192.168.0.2
+gateway = "i-did-not-read-the-docs" # e.g. 192.168.0.1
 
 ssh_public_keys = [
   # Put your public SSH key here

diff --git a/contrib/terraform/vsphere/main.tf b/contrib/terraform/vsphere/main.tf
@@ -19,7 +19,7 @@ data "vsphere_datastore" "datastore" {
 }
 
 data "vsphere_network" "network" {
-  name          = "VM Network"
+  name          = var.network
   datacenter_id = data.vsphere_datacenter.dc.id
 }
 
@@ -69,7 +69,7 @@ module "kubernetes" {
   pool_id      = vsphere_resource_pool.pool.id
   datastore_id = data.vsphere_datastore.datastore.id
 
-  folder                = ""
+  folder                = var.folder
   guest_id              = data.vsphere_virtual_machine.template.guest_id
   scsi_type             = data.vsphere_virtual_machine.template.scsi_type
   network_id            = data.vsphere_network.network.id

diff --git a/contrib/terraform/vsphere/modules/kubernetes-cluster/main.tf b/contrib/terraform/vsphere/modules/kubernetes-cluster/main.tf
@@ -5,21 +5,23 @@ resource "vsphere_virtual_machine" "worker" {
     if machine.node_type == "worker"
   }
 
-  name                = each.key
+  name = "${var.prefix}-${each.key}"
+
   resource_pool_id    = var.pool_id
   datastore_id        = var.datastore_id
 
   num_cpus            = var.worker_cores
   memory              = var.worker_memory
   memory_reservation  = var.worker_memory
   guest_id            = var.guest_id
-  enable_disk_uuid    = "true"
+  enable_disk_uuid    = "true" # needed for CSI provider
   scsi_type           = var.scsi_type
   folder              = var.folder
   firmware            = var.firmware
   hardware_version    = var.hardware_version
 
   wait_for_guest_net_routable = false
+  wait_for_guest_net_timeout = 0
 
   network_interface {
     network_id        = var.network_id
@@ -47,6 +49,7 @@ resource "vsphere_virtual_machine" "worker" {
   vapp {
     properties = {
       "user-data" = base64encode(templatefile("${path.module}/templates/cloud-init.tmpl", { ip = each.value.ip,
+                                                                  netmask = each.value.netmask,
                                                                   gw = var.gateway,
                                                                   dns = var.dns_primary,
                                                                   ssh_public_keys = var.ssh_public_keys}))
@@ -61,20 +64,24 @@ resource "vsphere_virtual_machine" "master" {
     if machine.node_type == "master"
   }
 
-  name                = each.key
+  name = "${var.prefix}-${each.key}"
+
   resource_pool_id    = var.pool_id
   datastore_id        = var.datastore_id
 
   num_cpus            = var.master_cores
   memory              = var.master_memory
   memory_reservation  = var.master_memory
   guest_id            = var.guest_id
-  enable_disk_uuid    = "true"
+  enable_disk_uuid    = "true" # needed for CSI provider
   scsi_type           = var.scsi_type
   folder              = var.folder
   firmware            = var.firmware
   hardware_version    = var.hardware_version
 
+  wait_for_guest_net_routable = false
+  wait_for_guest_net_timeout = 0
+
   network_interface {
     network_id        = var.network_id
     adapter_type      = var.adapter_type
@@ -101,6 +108,7 @@ resource "vsphere_virtual_machine" "master" {
   vapp {
     properties = {
       "user-data" = base64encode(templatefile("${path.module}/templates/cloud-init.tmpl", { ip = each.value.ip,
+                                                                  netmask = each.value.netmask,
                                                                   gw = var.gateway,
                                                                   dns = var.dns_primary,
                                                                   ssh_public_keys = var.ssh_public_keys}))

diff --git a/contrib/terraform/vsphere/modules/kubernetes-cluster/output.tf b/contrib/terraform/vsphere/modules/kubernetes-cluster/output.tf
@@ -1,13 +1,16 @@
 output "master_ip" {
   value = {
-    for instance in vsphere_virtual_machine.master :
-    instance.name => instance.default_ip_address
+    for name, machine in var.machines :
+    name => machine.ip
+    if machine.node_type == "master"
   }
 }
 
 output "worker_ip" {
   value = {
-    for instance in vsphere_virtual_machine.worker :
-    instance.name => instance.default_ip_address
+    for name, machine in var.machines :
+    name => machine.ip
+    if machine.node_type == "worker"
   }
 }
+
diff --git a/contrib/terraform/vsphere/modules/kubernetes-cluster/templates/cloud-init.tmpl b/contrib/terraform/vsphere/modules/kubernetes-cluster/templates/cloud-init.tmpl
@@ -25,7 +25,7 @@ write_files:
           ens192:
             dhcp4: false #true to use dhcp
             addresses:
-            - ${ip}
+            - ${ip}/${netmask}
             gateway4: ${gw} # Set gw here
             nameservers:
               addresses:

diff --git a/contrib/terraform/vsphere/modules/kubernetes-cluster/variables.tf b/contrib/terraform/vsphere/modules/kubernetes-cluster/variables.tf
@@ -5,7 +5,8 @@ variable "machines" {
   description = "Cluster machines"
   type = map(object({
     node_type = string
-    ip      = string
+    ip        = string
+    netmask   = string
   }))
 }
 

diff --git a/contrib/terraform/vsphere/output.tf b/contrib/terraform/vsphere/output.tf
@@ -23,7 +23,7 @@ output "vsphere_network" {
 }
 
 output "vsphere_folder" {
-  value = terraform.workspace
+  value = var.folder
 }
 
 output "vsphere_pool" {

diff --git a/contrib/terraform/vsphere/variables.tf b/contrib/terraform/vsphere/variables.tf
@@ -1,35 +1,20 @@
 ## Global ##
 
-variable "prefix" {
-  default = ""
-}
+# Required variables
 
 variable "machines" {
   description = "Cluster machines"
   type = map(object({
     node_type = string
     ip        = string
+    netmask   = string
   }))
 }
 
-variable "inventory_file" {
-  default = "inventory.ini"
-}
-
-variable "network" {
-  default = "VM Network"
-}
+variable "network" {}
 
 variable "gateway" {}
 
-variable "dns_primary" {
-  default = "8.8.4.4"
-}
-
-variable "dns_secondary" {
-  default = "8.8.8.8"
-}
-
 variable "vsphere_datacenter" {}
 
 variable "vsphere_compute_cluster" {}
@@ -44,21 +29,41 @@ variable "vsphere_server" {}
 
 variable "vsphere_hostname" {}
 
-variable "firmware" {
-  default = "bios"
+variable "ssh_public_keys" {
+  description = "List of public SSH keys which are injected into the VMs."
+  type        = list(string)
 }
 
-variable "hardware_version" {
-  default = "15"
+variable "template_name" {}
+
+# Optional variables (ones where reasonable defaults exist)
+
+variable "folder" {
+  default = ""
 }
 
-variable "template_name" {
-  default = "ubuntu-focal-20.04-cloudimg"
+variable "prefix" {
+  default = "k8s"
 }
 
-variable "ssh_public_keys" {
-  description = "List of public SSH keys which are injected into the VMs."
-  type        = list(string)
+variable "inventory_file" {
+  default = "inventory.ini"
+}
+
+variable "dns_primary" {
+  default = "8.8.4.4"
+}
+
+variable "dns_secondary" {
+  default = "8.8.8.8"
+}
+
+variable "firmware" {
+  default = "bios"
+}
+
+variable "hardware_version" {
+  default = "15"
 }
 
 ## Master ##