MicroK8S on Proxmox via LXC

Requirements

Name	Version
terraform	>= 1.0.10
proxmox	3.0.1-rc4
random	>= 3.1.0

Providers

Name	Version
external	n/a
null	n/a
proxmox	3.0.1-rc4
template	n/a
tls	n/a

Modules

No modules.

Resources

Name	Type
null_resource.argocd_install	resource
null_resource.microk8s_add_node	resource
null_resource.mountpoint_permission	resource
proxmox_lxc.microk8s	resource
tls_private_key.private_key	resource
external_external.microk8s_api_token	data source
external_external.microk8s_join_token	data source
template_file.argocd_install	data source
template_file.master_node_join	data source
template_file.post_create_sh	data source
template_file.rc_local_sh	data source

Inputs

Name	Description	Type	Default	Required
add_cluster_nodes	The list of VMIDs to add to the Kubernetes cluster.	list(string)	[]	no
cluster_addons	The addons to enable for the Kubernetes cluster.	object({ ingress = bool argocd = object({ enabled = bool admin_password = string ingress_host = string }) })	{ "argocd": { "admin_password": "", "enabled": false, "ingress_host": "" }, "ingress": true }	no
master_vmid	The VMID to assign to the Kubernetes master node.	number	null	no
mountpoints	The mountpoints to create on the Kubernetes cluster nodes.	list(object({ storage = string size = string mountpoint = string }))	[]	no
network	The network configuration for the Kubernetes cluster.	object({ name = string bridge = string gw = string ip = string })	n/a	yes
node_name	The name to assign to the Kubernetes cluster nodes.	string	n/a	yes
ostemplate	The proxmox template to use for the Kubernetes cluster nodes.	string	"local:vztmpl/ubuntu-23.10-standard_23.10-1_amd64.tar.zst"	no
proxmox_ssh	The network configuration for the Kubernetes cluster.	object({ host = string username = string password = string })	n/a	yes
resources	The resources to allocate to the Kubernetes cluster nodes.	object({ cores = number memory = number })	{ "cores": 4, "memory": 8192 }	no
rootfs	The root filesystem configuration for the Kubernetes cluster nodes.	object({ storage = string size = string })	{ "size": "80G", "storage": "local-lvm" }	no
ssh_public_keys	The SSH public keys to add to the Kubernetes cluster nodes.	list(string)	n/a	yes
tags	The tags to assign to the Kubernetes cluster nodes.	list(string)	[]	no
target_node	The proxmox target node to deploy the Kubernetes cluster to.	string	n/a	yes
vmid	The VMID to assign to the Kubernetes master node.	number	n/a	yes

Outputs

Name	Description
add_node_token	The URL of the Ingress controller
ingress_url	The URL of the Ingress controller
ip	The IP address of the Kubernetes node
kubernetes_api_url	The URL of the Ingress controller
kubernetes_token	The URL of the Ingress controller
vmid	The proxmox VMID of the node

Examples

Single Node

module "microk8s_master_node" {
  source      = "github.com/brunocarvalhodearaujo/microk8s-lxc-proxmox"
  target_node = "b550m"
  vmid        = 402
  node_name   = "microk8s-master"
  ostemplate  = "local:vztmpl/ubuntu-23.10-standard_23.10-1_amd64.tar.zst"
  ssh_public_keys = [
    var.public_key_openssh,
  ]
  cluster_addons = {
    ingress = true
    argocd = {
      admin_password = "loremipsum"
      enabled        = true
      ingress_host   = "argocd.example.com"
    }
  }
  resources = {
    cores  = 4
    memory = 4096
  }
  rootfs = {
    storage = "local-lvm"
    size    = "80G"
  }
  proxmox_ssh = {
    host     = var.proxmox_host
    username = var.proxmox_username
    password = var.proxmox_password
  }
  network = {
    name   = "eth0"
    bridge = "vmbr0"
    ip6    = "dhcp"
    gw     = "192.168.1.1"
    ip     = "192.168.1.50/24"
  }
}

Multi Node

resource "tls_private_key" "private_key" {
  algorithm = "RSA"
  rsa_bits  = 4096
}

module "microk8s_worker_node" {
  source      = "github.com/brunocarvalhodearaujo/microk8s-lxc-proxmox"
  target_node = "b550m"
  vmid        = 403
  node_name   = "microk8s-worker"
  ostemplate  = "local:vztmpl/ubuntu-23.10-standard_23.10-1_amd64.tar.zst"
  ssh_public_keys = [
    tls_private_key.private_key.public_key_openssh
  ]
  resources = {
    cores  = 4
    memory = 4096
  }
  rootfs = {
    storage = "local-lvm"
    size    = "80G"
  }
  proxmox_ssh = {
    host     = var.proxmox_host
    username = var.proxmox_username
    password = var.proxmox_password
  }
  network = {
    name   = "eth0"
    bridge = "vmbr0"
    ip6    = "dhcp"
    gw     = "192.168.1.1"
    ip     = "192.168.1.51/24"
  }
}

module "microk8s_master_node" {
  source      = "github.com/brunocarvalhodearaujo/microk8s-lxc-proxmox"
  target_node = "b550m"
  vmid        = 402
  node_name   = "microk8s-master"
  ostemplate  = "local:vztmpl/ubuntu-23.10-standard_23.10-1_amd64.tar.zst"
  ssh_public_keys = [
    tls_private_key.private_key.public_key_openssh
  ]
  cluster_addons = {
    ingress = true
    argocd = {
      admin_password = "loremipsum"
      enabled        = true
      ingress_host   = "argocd.example.com"
    }
  }
  resources = {
    cores  = 4
    memory = 8192
  }
  rootfs = {
    storage = "local-lvm"
    size    = "80G"
  }
  proxmox_ssh = {
    host     = var.proxmox_host
    username = var.proxmox_username
    password = var.proxmox_password
  }
  add_cluster_nodes = [
    nonsensitive(module.microk8s_worker_node.add_node_token)
  ]
  network = {
    name   = "eth0"
    bridge = "vmbr0"
    ip6    = "dhcp"
    gw     = "192.168.1.1"
    ip     = "192.168.1.50/24"
  }
  depends_on = [
    module.microk8s_worker_node
  ]
}

Integration with kubernetes provider

terraform {
  required_version = ">= 1.0.10"

  required_providers {
    proxmox = {
      source  = "Telmate/proxmox"
      version = "3.0.1-rc4"
    }
    kubernetes = {
      source  = "hashicorp/kubernetes"
      version = "2.33.0"
    }
  }
}

provider "proxmox" {}

module "microk8s_master_node" {
    // put the module configuration here
}

provider "kubernetes" {
  host     = module.microk8s_master_node.kubernetes_api_url
  token    = module.microk8s_master_node.kubernetes_token
  insecure = true
}

resource "kubernetes_namespace" "sample" {
  metadata {
    name = "my-namespace"
  }
}

Integration with cloudflare provider

terraform {
  required_version = ">= 1.0.10"

  required_providers {
    random = {
      source  = "hashicorp/random"
      version = ">= 3.1.0"
    }
    proxmox = {
      source  = "Telmate/proxmox"
      version = "3.0.1-rc4"
    }
    cloudflare = {
      source  = "cloudflare/cloudflare"
      version = ">= 4.44.0"
    }
    kubernetes = {
      source  = "hashicorp/kubernetes"
      version = "2.33.0"
    }
  }
}

provider "proxmox" {}

module "microk8s_master_node" {
  // put the module configuration here
}

data "cloudflare_zone" "example" {
  name = "example.com"
}

resource "random_password" "tunnel_secret" {
  length = 64
}

resource "cloudflare_zero_trust_tunnel_cloudflared" "example" {
  account_id = var.cloudflare_account_id
  name       = "example"
  secret     = base64sha256(random_password.tunnel_secret.result)
}

resource "cloudflare_record" "microk8s" {
  zone_id = data.cloudflare_zone.example.id
  name    = "*"
  content = cloudflare_zero_trust_tunnel_cloudflared.example.cname
  type    = "CNAME"
  proxied = true
}

resource "cloudflare_zero_trust_tunnel_cloudflared_config" "b550mk_tunnel" {
  tunnel_id  = cloudflare_zero_trust_tunnel_cloudflared.b550mk.id
  account_id = var.cloudflare_account_id
  config {
    warp_routing {
      enabled = true
    }

    ingress_rule {
      origin_request {
        http2_origin  = true
        no_tls_verify = true
      }
      hostname = cloudflare_record.example.hostname
      service  = module.microk8s_master_node.ingress_url
    }

    ingress_rule {
      service = "http_status:404"
    }
  }
}

Then you can access the Kubernetes cluster via the cloudflare record *.example.com and the argo web interface via the cloudflare record argocd.example.com.

Ingress controller is enabled by default, so you can access the services via the cloudflare record *.example.com.

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: my-sample-ingress
  namespace: my-namespace
  annotations:
    nginx.ingress.kubernetes.io/ssl-redirect: "false"
    nginx.ingress.kubernetes.io/backend-protocol: "HTTP"
spec:
  ingressClassName: "nginx"
  rules:
    - host: test.example.com
      http:
        paths:
          - pathType: Prefix
            path: /
            backend:
              service:
                name: sample-service
                port:
                  name: http

References

• automated kubernetes proxmox
• Installing microk8s in an LXC container