From b36d59c7620d31e7d199383a9a7a06e723cd85bb Mon Sep 17 00:00:00 2001 From: Christian Schlotter Date: Wed, 29 May 2024 17:19:03 +0200 Subject: [PATCH] k8s-infra-vsphere: add terraform to provision vsphere for ci --- .../k8s-infra-vsphere/vsphere/folders.tf | 39 ++++++++ .../vsphere/hack/ensure_ova_from_github.sh | 59 +++++++++++ .../k8s-infra-vsphere/vsphere/permissions.tf | 99 +++++++++++++++++++ .../vsphere/resource-pools.tf | 35 +++++++ .../k8s-infra-vsphere/vsphere/vm-templates.tf | 49 +++++++++ 5 files changed, 281 insertions(+) create mode 100644 infra/gcp/terraform/k8s-infra-vsphere/vsphere/folders.tf create mode 100755 infra/gcp/terraform/k8s-infra-vsphere/vsphere/hack/ensure_ova_from_github.sh create mode 100644 infra/gcp/terraform/k8s-infra-vsphere/vsphere/permissions.tf create mode 100644 infra/gcp/terraform/k8s-infra-vsphere/vsphere/resource-pools.tf create mode 100644 infra/gcp/terraform/k8s-infra-vsphere/vsphere/vm-templates.tf diff --git a/infra/gcp/terraform/k8s-infra-vsphere/vsphere/folders.tf b/infra/gcp/terraform/k8s-infra-vsphere/vsphere/folders.tf new file mode 100644 index 00000000000..2541f17c8f7 --- /dev/null +++ b/infra/gcp/terraform/k8s-infra-vsphere/vsphere/folders.tf @@ -0,0 +1,39 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +resource "vsphere_folder" "cpi" { + path = "cloud-provider-vsphere" + type = "vm" + datacenter_id = data.vsphere_datacenter.datacenter.id +} + +resource "vsphere_folder" "capi" { + path = "cluster-api-provider-vsphere" + type = "vm" + datacenter_id = data.vsphere_datacenter.datacenter.id +} + +resource "vsphere_folder" "image-builder" { + path = "image-builder" + type = "vm" + datacenter_id = data.vsphere_datacenter.datacenter.id +} + +resource "vsphere_folder" "templates" { + path = "templates" + type = "vm" + datacenter_id = data.vsphere_datacenter.datacenter.id +} diff --git a/infra/gcp/terraform/k8s-infra-vsphere/vsphere/hack/ensure_ova_from_github.sh b/infra/gcp/terraform/k8s-infra-vsphere/vsphere/hack/ensure_ova_from_github.sh new file mode 100755 index 00000000000..e70bf67ac74 --- /dev/null +++ b/infra/gcp/terraform/k8s-infra-vsphere/vsphere/hack/ensure_ova_from_github.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +if [[ "${DEBUG}" == "true" ]]; then + set -x +fi + +# TODO: ensure govc +# TODO: ensure all env vars are set +# TODO: set pipefail, etc. + +GITHUB_CA_CERTIFICATE_JSON='{"cert_chain": {"cert_chain": ["'${GITHUB_CA_CERTIFICATE}'"]}}' + +# Ensure the githubs CA certificate exists + +echo "> Ensuring githubs CA certificate exists so library imports work" + +if [[ "$(govc session.login -r -X GET "/api/vcenter/certificate-management/vcenter/trusted-root-chains" | grep -e "${GITHUB_CA_THUMBPRINT}")" == "" ]]; then + govc session.login -r -X POST "/api/vcenter/certificate-management/vcenter/trusted-root-chains" <<< ${GITHUB_CA_CERTIFICATE_JSON} +fi + +function ensureOVA() { + URL="${1}" + NAME=${URL##*/} + NAME=${NAME%.ova} + + echo "> Ensuring OVA ${NAME} from ${URL} exists" + + if [[ "$(govc library.info "/${CONTENT_LIBRARY_NAME}/${NAME}" || true)" == "" ]]; then + echo ">> in content library /${CONTENT_LIBRARY_NAME}" + govc library.import -pull "/${CONTENT_LIBRARY_NAME}" "${URL}" + fi + + if [[ "$(govc vm.info "${TEMPLATES_FOLDER}/${NAME}" || true)" == "" ]]; then + echo ">> as VM template in ${TEMPLATES_FOLDER}" + govc library.deploy -folder "${TEMPLATES_FOLDER}" -ds "${DATASTORE}" -pool "${RESOURCE_POOL}" "/${CONTENT_LIBRARY_NAME}/${NAME}" + govc snapshot.create -vm "${TEMPLATES_FOLDER}/${NAME}" "for-linkedclone" + govc vm.markastemplate "${TEMPLATES_FOLDER}/${NAME}" + fi +} + +echo "> Ensuring OVA from ${URL}" + +ensureOVA "${URL}" diff --git a/infra/gcp/terraform/k8s-infra-vsphere/vsphere/permissions.tf b/infra/gcp/terraform/k8s-infra-vsphere/vsphere/permissions.tf new file mode 100644 index 00000000000..178caf069ce --- /dev/null +++ b/infra/gcp/terraform/k8s-infra-vsphere/vsphere/permissions.tf @@ -0,0 +1,99 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +resource "vsphere_role" "capv-ci-content-library" { + name = "capv-ci-content-library" + role_privileges = [ + "ContentLibrary.DownloadSession", + "ContentLibrary.ReadStorage", + "ContentLibrary.SyncLibraryItem", + ] +} + +resource "vsphere_role" "capv-ci" { + name = "capv-ci" + role_privileges = [ + "Cns.Searchable", + "Datastore.AllocateSpace", + "Datastore.Browse", + "Datastore.FileManagement", + "Folder.Create", + "Folder.Delete", + "Global.SetCustomField", + "Network.Assign", + "Resource.AssignVMToPool", + "Resource.CreatePool", + "Resource.DeletePool", + "Sessions.GlobalMessage", + "Sessions.ValidateSession", + "StorageProfile.View", + "VApp.ApplicationConfig", + "VApp.Import", + "VApp.InstanceConfig", + "VirtualMachine.Config.AddExistingDisk", + "VirtualMachine.Config.AddNewDisk", + "VirtualMachine.Config.AddRemoveDevice", + "VirtualMachine.Config.AdvancedConfig", + "VirtualMachine.Config.Annotation", + "VirtualMachine.Config.CPUCount", + "VirtualMachine.Config.ChangeTracking", + "VirtualMachine.Config.DiskExtend", + "VirtualMachine.Config.EditDevice", + "VirtualMachine.Config.HostUSBDevice", + "VirtualMachine.Config.ManagedBy", + "VirtualMachine.Config.Memory", + "VirtualMachine.Config.RawDevice", + "VirtualMachine.Config.RemoveDisk", + "VirtualMachine.Config.Resource", + "VirtualMachine.Config.Settings", + "VirtualMachine.Config.SwapPlacement", + "VirtualMachine.Config.UpgradeVirtualHardware", + "VirtualMachine.Interact.ConsoleInteract", + "VirtualMachine.Interact.DeviceConnection", + "VirtualMachine.Interact.PowerOff", + "VirtualMachine.Interact.PowerOn", + "VirtualMachine.Interact.SetCDMedia", + "VirtualMachine.Interact.SetFloppyMedia", + "VirtualMachine.Inventory.Create", + "VirtualMachine.Inventory.CreateFromExisting", + "VirtualMachine.Inventory.Delete", + "VirtualMachine.Provisioning.Clone", + "VirtualMachine.Provisioning.CloneTemplate", + "VirtualMachine.Provisioning.CreateTemplateFromVM", + "VirtualMachine.Provisioning.DeployTemplate", + "VirtualMachine.Provisioning.DiskRandomRead", + "VirtualMachine.Provisioning.GetVmFiles", + "VirtualMachine.State.CreateSnapshot", + "VirtualMachine.State.RemoveSnapshot", + ] +} + +# resource "vsphere_entity_permissions" "rp-capv" { +# entity_id = vsphere_resource_pool.capi.id +# entity_type = "ResourcePool" +# permissions { +# user_or_group = "vsphere.local\\DCClients" +# propagate = true +# is_group = true +# role_id = data.vsphere_role.role1.id +# } +# permissions { +# user_or_group = "vsphere.local\\ExternalIDPUsers" +# propagate = true +# is_group = true +# role_id = vsphere_role.role2.id +# } +# } diff --git a/infra/gcp/terraform/k8s-infra-vsphere/vsphere/resource-pools.tf b/infra/gcp/terraform/k8s-infra-vsphere/vsphere/resource-pools.tf new file mode 100644 index 00000000000..d3ed3c4b2ff --- /dev/null +++ b/infra/gcp/terraform/k8s-infra-vsphere/vsphere/resource-pools.tf @@ -0,0 +1,35 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +resource "vsphere_resource_pool" "cpi" { + name = "cloud-provider-vsphere" + parent_resource_pool_id = data.vsphere_compute_cluster.compute_cluster.resource_pool_id +} + +resource "vsphere_resource_pool" "capi" { + name = "cluster-api-provider-vsphere" + parent_resource_pool_id = data.vsphere_compute_cluster.compute_cluster.resource_pool_id +} + +resource "vsphere_resource_pool" "image-builder" { + name = "image-builder" + parent_resource_pool_id = data.vsphere_compute_cluster.compute_cluster.resource_pool_id +} + +resource "vsphere_resource_pool" "templates" { + name = "templates" + parent_resource_pool_id = data.vsphere_compute_cluster.compute_cluster.resource_pool_id +} diff --git a/infra/gcp/terraform/k8s-infra-vsphere/vsphere/vm-templates.tf b/infra/gcp/terraform/k8s-infra-vsphere/vsphere/vm-templates.tf new file mode 100644 index 00000000000..cb8ed72fa8c --- /dev/null +++ b/infra/gcp/terraform/k8s-infra-vsphere/vsphere/vm-templates.tf @@ -0,0 +1,49 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +resource "vsphere_content_library" "capv" { + name = "capv" + description = "Content Library for CAPV." + storage_backing = [data.vsphere_datastore.datastore.id] +} + +# TODO: consider if this should be pure bash instead? + +resource "terraform_data" "ova_templates" { + depends_on = [vsphere_content_library.capv] + + triggers_replace = sha512(file("${path.module}/hack/ensure_ova_from_github.sh")) + + provisioner "local-exec" { + when = create + command = "${path.module}/hack/ensure_ova_from_github.sh" + environment = { + "GOVC_URL" = "${var.vsphere_user}:${var.vsphere_password}@${var.vsphere_server}" + "GOVC_INSECURE" = "true" + "DEBUG" = "true" + "GITHUB_CA_CERTIFICATE" = "${var.github_ca_certificate}" + "GITHUB_CA_THUMBPRINT" = "${var.github_ca_thumbprint}" + "CONTENT_LIBRARY_NAME" = "${vsphere_content_library.capv.name}" + "TEMPLATES_FOLDER" = "${vsphere_folder.templates.path}" + "DATASTORE" = "${data.vsphere_datastore.datastore.name}" + "RESOURCE_POOL" = "/${data.vsphere_datacenter.datacenter.name}/host/${data.vsphere_compute_cluster.compute_cluster.name}/Resources/${vsphere_resource_pool.templates.name}" + "URL" = "${each.value}" + } + interpreter = ["/bin/bash", "-c"] + } + + for_each = var.ova_templates +}