From 6f8f0e9ba2ec82be83f6ac62b3fe712e4878eaf5 Mon Sep 17 00:00:00 2001 From: Lars Larsson Date: Wed, 10 Nov 2021 10:27:33 +0100 Subject: [PATCH 1/4] Fixes various issues in vSphere Terraform code Provided to address various shortcomings and to fix the following issue in upstream Kubespray: https://github.com/kubernetes-sigs/kubespray/issues/8176 --- contrib/terraform/vsphere/README.md | 28 ++++++--- contrib/terraform/vsphere/default.tfvars | 15 +++-- contrib/terraform/vsphere/main.tf | 4 +- .../modules/kubernetes-cluster/main.tf | 16 +++-- .../modules/kubernetes-cluster/output.tf | 11 ++-- .../templates/cloud-init.tmpl | 2 +- .../modules/kubernetes-cluster/variables.tf | 3 +- contrib/terraform/vsphere/output.tf | 2 +- contrib/terraform/vsphere/variables.tf | 59 ++++++++++--------- 9 files changed, 87 insertions(+), 53 deletions(-) diff --git a/contrib/terraform/vsphere/README.md b/contrib/terraform/vsphere/README.md index dfb53485c91..7dd73f1db0b 100644 --- a/contrib/terraform/vsphere/README.md +++ b/contrib/terraform/vsphere/README.md @@ -1,6 +1,6 @@ -# Kubernetes on Exoscale with Terraform +# Kubernetes on vSphere with Terraform -Provision a Kubernetes cluster on [vSphere](https://www.vmware.com/se/products/vsphere.html) using Terraform and Kubespray. +Provision a Kubernetes cluster on [vSphere](https://www.vmware.com/products/vsphere.html) using Terraform and Kubespray. ## Overview @@ -98,20 +98,32 @@ ansible-playbook -i inventory.ini ../../cluster.yml -b -v * `machines`: Machines to provision. Key of this object will be used as the name of the machine * `node_type`: The role of this node *(master|worker)* - * `ip`: The IP address with the netmask (CIDR notation) + * `ip`: The IP address of the machine + * `netmask`: The netmask to use (to be used on the right hand side in CIDR notation, e.g., `24`) +* `network`: The name of the network to attach the machines to * `gateway`: The IP address of the network gateway -* `ssh_public_keys`: List of public SSH keys to install on all machines * `vsphere_datacenter`: The identifier of vSphere data center * `vsphere_compute_cluster`: The identifier of vSphere compute cluster * `vsphere_datastore`: The identifier of vSphere data store * `vsphere_server`: The address of vSphere server * `vsphere_hostname`: The IP address of vSphere hostname -* `template_name`: The name of a base image (the image has to be uploaded to vSphere beforehand) +* `ssh_public_keys`: List of public SSH keys to install on all machines +* `template_name`: The name of a base image (the OVF template be defined in vSphere beforehand) ### Optional -* `prefix`: Prefix to use for all resources, required to be unique for all clusters in the same project *(Defaults to `default`)* -* `dns_primary`: The IP address of primary DNS server *(Defaults to `8.8.4.4`)* -* `dns_secondary`:The IP address of secondary DNS server *(Defaults to `8.8.8.8`)* +* `folder`: Name of the folder to put all machines in (default: `""`) +* `prefix`: Prefix to use for all resources, required to be unique for all clusters in the same project (default: `""`) +* `inventory_file`: Name of the generated inventory file for Kubespray to use in the Ansible step (default: `inventory.ini`) +* `dns_primary`: The IP address of primary DNS server (default: `8.8.4.4`) +* `dns_secondary`: The IP address of secondary DNS server (default: `8.8.8.8`) +* `firmware`: Firmware to use (default: `bios`) +* `hardware_version`: The version of the hardware (default: `15`) +* `master_cores`: The number of CPU cores for the master nodes (default: 4) +* `master_memory`: The amount of RAM for the master nodes in MB (default: 4096) +* `master_disk_size`: The amount of disk space for the master nodes in GB (default: 20) +* `worker_cores`: The number of CPU cores for the worker nodes (default: 16) +* `worker_memory`: The amount of RAM for the worker nodes in MB (default: 8192) +* `worker_disk_size`: The amount of disk space for the worker nodes in GB (default: 100) An example variables file can be found `default.tfvars` diff --git a/contrib/terraform/vsphere/default.tfvars b/contrib/terraform/vsphere/default.tfvars index f0ee03ed846..a3f7a2e77a7 100644 --- a/contrib/terraform/vsphere/default.tfvars +++ b/contrib/terraform/vsphere/default.tfvars @@ -1,23 +1,28 @@ -prefix = "default" +prefix = "k8s" inventory_file = "inventory.ini" +network = "VM Network" + machines = { "master-0" : { "node_type" : "master", - "ip" : "i-did-not-read-the-docs" # e.g. 192.168.0.2/24 + "ip" : "i-did-not-read-the-docs", # e.g. 192.168.0.10 + "netmask": "24" }, "worker-0" : { "node_type" : "worker", - "ip" : "i-did-not-read-the-docs" # e.g. 192.168.0.2/24 + "ip" : "i-did-not-read-the-docs", # e.g. 192.168.0.20 + "netmask": "24" }, "worker-1" : { "node_type" : "worker", - "ip" : "i-did-not-read-the-docs" # e.g. 192.168.0.2/24 + "ip" : "i-did-not-read-the-docs", # e.g. 192.168.0.21 + "netmask": "24" } } -gateway = "i-did-not-read-the-docs" # e.g. 192.168.0.2 +gateway = "i-did-not-read-the-docs" # e.g. 192.168.0.1 ssh_public_keys = [ # Put your public SSH key here diff --git a/contrib/terraform/vsphere/main.tf b/contrib/terraform/vsphere/main.tf index 41eff417073..b8c52a1f57e 100644 --- a/contrib/terraform/vsphere/main.tf +++ b/contrib/terraform/vsphere/main.tf @@ -19,7 +19,7 @@ data "vsphere_datastore" "datastore" { } data "vsphere_network" "network" { - name = "VM Network" + name = var.network datacenter_id = data.vsphere_datacenter.dc.id } @@ -69,7 +69,7 @@ module "kubernetes" { pool_id = vsphere_resource_pool.pool.id datastore_id = data.vsphere_datastore.datastore.id - folder = "" + folder = var.folder guest_id = data.vsphere_virtual_machine.template.guest_id scsi_type = data.vsphere_virtual_machine.template.scsi_type network_id = data.vsphere_network.network.id diff --git a/contrib/terraform/vsphere/modules/kubernetes-cluster/main.tf b/contrib/terraform/vsphere/modules/kubernetes-cluster/main.tf index ecb3165b8ff..97b2a2694d3 100644 --- a/contrib/terraform/vsphere/modules/kubernetes-cluster/main.tf +++ b/contrib/terraform/vsphere/modules/kubernetes-cluster/main.tf @@ -5,7 +5,8 @@ resource "vsphere_virtual_machine" "worker" { if machine.node_type == "worker" } - name = each.key + name = "${var.prefix}-${each.key}" + resource_pool_id = var.pool_id datastore_id = var.datastore_id @@ -13,13 +14,14 @@ resource "vsphere_virtual_machine" "worker" { memory = var.worker_memory memory_reservation = var.worker_memory guest_id = var.guest_id - enable_disk_uuid = "true" + enable_disk_uuid = "true" # needed for CSI provider scsi_type = var.scsi_type folder = var.folder firmware = var.firmware hardware_version = var.hardware_version wait_for_guest_net_routable = false + wait_for_guest_net_timeout = 0 network_interface { network_id = var.network_id @@ -47,6 +49,7 @@ resource "vsphere_virtual_machine" "worker" { vapp { properties = { "user-data" = base64encode(templatefile("${path.module}/templates/cloud-init.tmpl", { ip = each.value.ip, + netmask = each.value.netmask, gw = var.gateway, dns = var.dns_primary, ssh_public_keys = var.ssh_public_keys})) @@ -61,7 +64,8 @@ resource "vsphere_virtual_machine" "master" { if machine.node_type == "master" } - name = each.key + name = "${var.prefix}-${each.key}" + resource_pool_id = var.pool_id datastore_id = var.datastore_id @@ -69,12 +73,15 @@ resource "vsphere_virtual_machine" "master" { memory = var.master_memory memory_reservation = var.master_memory guest_id = var.guest_id - enable_disk_uuid = "true" + enable_disk_uuid = "true" # needed for CSI provider scsi_type = var.scsi_type folder = var.folder firmware = var.firmware hardware_version = var.hardware_version + wait_for_guest_net_routable = false + wait_for_guest_net_timeout = 0 + network_interface { network_id = var.network_id adapter_type = var.adapter_type @@ -101,6 +108,7 @@ resource "vsphere_virtual_machine" "master" { vapp { properties = { "user-data" = base64encode(templatefile("${path.module}/templates/cloud-init.tmpl", { ip = each.value.ip, + netmask = each.value.netmask, gw = var.gateway, dns = var.dns_primary, ssh_public_keys = var.ssh_public_keys})) diff --git a/contrib/terraform/vsphere/modules/kubernetes-cluster/output.tf b/contrib/terraform/vsphere/modules/kubernetes-cluster/output.tf index e4f576cc0d2..a562f8c2586 100644 --- a/contrib/terraform/vsphere/modules/kubernetes-cluster/output.tf +++ b/contrib/terraform/vsphere/modules/kubernetes-cluster/output.tf @@ -1,13 +1,16 @@ output "master_ip" { value = { - for instance in vsphere_virtual_machine.master : - instance.name => instance.default_ip_address + for name, machine in var.machines : + name => machine.ip + if machine.node_type == "master" } } output "worker_ip" { value = { - for instance in vsphere_virtual_machine.worker : - instance.name => instance.default_ip_address + for name, machine in var.machines : + name => machine.ip + if machine.node_type == "worker" } } + diff --git a/contrib/terraform/vsphere/modules/kubernetes-cluster/templates/cloud-init.tmpl b/contrib/terraform/vsphere/modules/kubernetes-cluster/templates/cloud-init.tmpl index a7bd0e78571..eec9d923f67 100644 --- a/contrib/terraform/vsphere/modules/kubernetes-cluster/templates/cloud-init.tmpl +++ b/contrib/terraform/vsphere/modules/kubernetes-cluster/templates/cloud-init.tmpl @@ -25,7 +25,7 @@ write_files: ens192: dhcp4: false #true to use dhcp addresses: - - ${ip} + - ${ip}/${netmask} gateway4: ${gw} # Set gw here nameservers: addresses: diff --git a/contrib/terraform/vsphere/modules/kubernetes-cluster/variables.tf b/contrib/terraform/vsphere/modules/kubernetes-cluster/variables.tf index 5630fe9e1d9..756d6316b8b 100644 --- a/contrib/terraform/vsphere/modules/kubernetes-cluster/variables.tf +++ b/contrib/terraform/vsphere/modules/kubernetes-cluster/variables.tf @@ -5,7 +5,8 @@ variable "machines" { description = "Cluster machines" type = map(object({ node_type = string - ip = string + ip = string + netmask = string })) } diff --git a/contrib/terraform/vsphere/output.tf b/contrib/terraform/vsphere/output.tf index bdd67dc8309..a4338d9be3e 100644 --- a/contrib/terraform/vsphere/output.tf +++ b/contrib/terraform/vsphere/output.tf @@ -23,7 +23,7 @@ output "vsphere_network" { } output "vsphere_folder" { - value = terraform.workspace + value = var.folder } output "vsphere_pool" { diff --git a/contrib/terraform/vsphere/variables.tf b/contrib/terraform/vsphere/variables.tf index 346c10c13e5..dc1b817072a 100644 --- a/contrib/terraform/vsphere/variables.tf +++ b/contrib/terraform/vsphere/variables.tf @@ -1,35 +1,20 @@ ## Global ## -variable "prefix" { - default = "" -} +# Required variables variable "machines" { description = "Cluster machines" type = map(object({ node_type = string ip = string + netmask = string })) } -variable "inventory_file" { - default = "inventory.ini" -} - -variable "network" { - default = "VM Network" -} +variable "network" {} variable "gateway" {} -variable "dns_primary" { - default = "8.8.4.4" -} - -variable "dns_secondary" { - default = "8.8.8.8" -} - variable "vsphere_datacenter" {} variable "vsphere_compute_cluster" {} @@ -44,21 +29,41 @@ variable "vsphere_server" {} variable "vsphere_hostname" {} -variable "firmware" { - default = "bios" +variable "ssh_public_keys" { + description = "List of public SSH keys which are injected into the VMs." + type = list(string) } -variable "hardware_version" { - default = "15" +variable "template_name" {} + +# Optional variables (ones where reasonable defaults exist) + +variable "folder" { + default = "" } -variable "template_name" { - default = "ubuntu-focal-20.04-cloudimg" +variable "prefix" { + default = "" } -variable "ssh_public_keys" { - description = "List of public SSH keys which are injected into the VMs." - type = list(string) +variable "inventory_file" { + default = "inventory.ini" +} + +variable "dns_primary" { + default = "8.8.4.4" +} + +variable "dns_secondary" { + default = "8.8.8.8" +} + +variable "firmware" { + default = "bios" +} + +variable "hardware_version" { + default = "15" } ## Master ## From 5a13c8eabb5dfac0bf116a85dc3c3698cf681a01 Mon Sep 17 00:00:00 2001 From: Lars Larsson Date: Thu, 11 Nov 2021 08:27:38 +0100 Subject: [PATCH 2/4] Resolves Terraform formatting issues --- contrib/terraform/vsphere/default.tfvars | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/terraform/vsphere/default.tfvars b/contrib/terraform/vsphere/default.tfvars index a3f7a2e77a7..be465bf5a0f 100644 --- a/contrib/terraform/vsphere/default.tfvars +++ b/contrib/terraform/vsphere/default.tfvars @@ -8,17 +8,17 @@ machines = { "master-0" : { "node_type" : "master", "ip" : "i-did-not-read-the-docs", # e.g. 192.168.0.10 - "netmask": "24" + "netmask" : "24" }, "worker-0" : { "node_type" : "worker", "ip" : "i-did-not-read-the-docs", # e.g. 192.168.0.20 - "netmask": "24" + "netmask" : "24" }, "worker-1" : { "node_type" : "worker", "ip" : "i-did-not-read-the-docs", # e.g. 192.168.0.21 - "netmask": "24" + "netmask" : "24" } } From 842fc850b1eaa30d0117a7aad30c282fbb92b225 Mon Sep 17 00:00:00 2001 From: Lars Larsson Date: Fri, 12 Nov 2021 15:28:42 +0100 Subject: [PATCH 3/4] Sets default prefix to human-readable name --- contrib/terraform/vsphere/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/terraform/vsphere/variables.tf b/contrib/terraform/vsphere/variables.tf index dc1b817072a..94b98dac1ee 100644 --- a/contrib/terraform/vsphere/variables.tf +++ b/contrib/terraform/vsphere/variables.tf @@ -43,7 +43,7 @@ variable "folder" { } variable "prefix" { - default = "" + default = "k8s" } variable "inventory_file" { From 90cc1464a76baba696ce42ef71cdbc9ba674e931 Mon Sep 17 00:00:00 2001 From: Lars Larsson Date: Fri, 12 Nov 2021 15:51:56 +0100 Subject: [PATCH 4/4] Documents new default prefix in README --- contrib/terraform/vsphere/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/terraform/vsphere/README.md b/contrib/terraform/vsphere/README.md index 7dd73f1db0b..9373c56d6ef 100644 --- a/contrib/terraform/vsphere/README.md +++ b/contrib/terraform/vsphere/README.md @@ -113,7 +113,7 @@ ansible-playbook -i inventory.ini ../../cluster.yml -b -v ### Optional * `folder`: Name of the folder to put all machines in (default: `""`) -* `prefix`: Prefix to use for all resources, required to be unique for all clusters in the same project (default: `""`) +* `prefix`: Prefix to use for all resources, required to be unique for all clusters in the same project (default: `"k8s"`) * `inventory_file`: Name of the generated inventory file for Kubespray to use in the Ansible step (default: `inventory.ini`) * `dns_primary`: The IP address of primary DNS server (default: `8.8.4.4`) * `dns_secondary`: The IP address of secondary DNS server (default: `8.8.8.8`)