Support Puppet-managed LoadBalancers

.github/workflows/test.yaml

@@ -0,0 +1,19 @@
+name: Pull Request
+on:
+  pull_request:
+    branches:
+    - master
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Extract Terraform version from constraints in module
+        run: echo TF_VERSION=$(grep "^[[:space:]]\+required_version = \"" provider.tf | cut -d= -f2- | tr -d ' "') >> $GITHUB_ENV
+      - uses: hashicorp/setup-terraform@v1
+        with:
+          terraform_version: ${{ env.TF_VERSION }}
+      - run: terraform fmt -check -recursive
+      - run: terraform init -input=false
+      - run: terraform validate

files/commit-hieradata.sh

@@ -0,0 +1,75 @@
+#!/bin/sh
+
+readonly cluster_id=$1
+readonly branch="tf/lbaas/${cluster_id}"
+
+cd appuio_hieradata || exit 1
+
+git config user.name "${GIT_AUTHOR_NAME}"
+git config user.email "${GIT_AUTHOR_EMAIL}"
+
+# Checkout feature branch
+# 1. try to check out as tracking branch from origin
+# 2. checkout as new branch
+# 3. checkout existing local branch
+# For existing local branch, amend existing commit
+if ! git checkout -t origin/"${branch}"; then
+  if ! git checkout -b "${branch}"; then
+    git checkout "${branch}"
+  fi
+fi
+
+git add "lbaas/${cluster_id}.yaml"
+
+status=$(git status --porcelain)
+echo "'${status}'"
+
+commit_message="Update LBaaS hieradata for ${cluster_id}"
+push=1
+if [ "${status}"  = "M  lbaas/${cluster_id}.yaml" ]; then
+  git commit -m"${commit_message}"
+elif [ "${status}" != "" ]; then
+  # assume new hieradata
+  commit_message="Create LBaaS hieradata for ${cluster_id}"
+  git commit -m "${commit_message}"
+else
+  push=0
+fi
+
+if [ "${push}" -eq 1 ]; then
+  # Push branch to origin and set upstream
+  git push origin "${branch}"
+fi
+
+# Always set branch's upstream to origin/master.
+#
+# If we would set the branch's upstream to the pushed branch, subsequent
+# terraform runs break if the upstream branch has been merged or deleted.
+#
+# If we only set the upstream when pushing, subsequent terraform runs break if
+# there's been no changes in the hieradata.
+git branch -u origin/master
+
+# Create MR if none exists yet
+open_mrs=$(curl -sH "Authorization: Bearer ${HIERADATA_REPO_TOKEN}" \
+  "https://git.vshn.net/api/v4/projects/368/merge_requests?state=opened&source_branch=tf/lbaas/${cluster_id}")
+if [ "${push}" -eq 0 ]; then
+  mr_url="No changes, skipping push and MR creation"
+elif [ "${open_mrs}" = "[]" ]; then
+  # create MR
+  mr_url=$(curl -XPOST -sH "Authorization: Bearer ${HIERADATA_REPO_TOKEN}" \
+    -H"Content-type: application/json" \
+    "https://git.vshn.net/api/v4/projects/368/merge_requests" \
+    -d \
+    "{
+      \"id\": 368,
+      \"source_branch\": \"tf/lbaas/${cluster_id}\",
+      \"target_branch\": \"master\",
+      \"title\": \"${commit_message}\",
+      \"remove_source_branch\": true
+    }" | jq -r '.web_url')
+else
+  mr_url=$(echo "${open_mrs}" | jq -r '.[0].web_url')
+fi
+
+echo "${mr_url}" > /tf/.mr_url.txt

files/register-server.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+set -eo pipefail
+
+curl -s -X POST -H "X-AccessToken: ${CONTROL_VSHN_NET_TOKEN}" \
+  https://control.vshn.net/api/servers/1/appuio/ \
+  -d "{
+    \"customer\": \"appuio\",
+    \"fqdn\": \"${SERVER_FQDN}\",
+    \"location\": \"cloudscale\",
+    \"region\": \"${SERVER_REGION}\",
+    \"environment\": \"AppuioLbaas\",
+    \"project\": \"lbaas\",
+    \"role\": \"lb\",
+    \"stage\": \"${CLUSTER_ID}\"
+  }"

lb.tf

@@ -15,76 +15,154 @@ resource "cloudscale_server_group" "lb" {
   zone_slug = "${var.region}1"
 }
 
+locals {
+  instance_fqdns = formatlist("%s.${local.node_name_suffix}", random_id.lb[*].hex)
+
+  common_user_data = {
+    "package_update"  = true,
+    "package_upgrade" = true,
+    "runcmd" = [
+      "sleep '5'",
+      "wget -O /tmp/puppet-source.deb https://apt.puppetlabs.com/puppet6-release-focal.deb",
+      "dpkg -i /tmp/puppet-source.deb",
+      "rm /tmp/puppet-source.deb",
+      "apt-get update",
+      "apt-get -y install puppet-agent",
+      "apt-get -y purge snapd",
+      "mkdir -p /etc/puppetlabs/facter/facts.d",
+      "netplan apply",
+      ["bash", "-c",
+      "set +e -x; for ((i=0; i < 3; ++i)); do /opt/puppetlabs/bin/puppet facts && break; done; for ((i=0; i < 3; ++i)); do /opt/puppetlabs/bin/puppet agent -t --server master.puppet.vshn.net && break; done"],
+      "sleep 5",
+      "shutdown --reboot +1 'Reboot for system setup'",
+    ],
+    "manage_etc_hosts" = true,
+    "write_files" = [
+      {
+        path       = "/etc/netplan/60-ens4.yaml"
+        "encoding" = "b64"
+        "content" = base64encode(yamlencode({
+          "network" = {
+            "ethernets" = {
+              "ens4" = {
+                "dhcp4" = true,
+              },
+            },
+            "version" = 2,
+          }
+        }))
+      }
+    ]
+  }
+}
+
+resource "null_resource" "register_lb" {
+  triggers = {
+    # Refresh resource when script changes -- this is probaby not required for production
+    script_sha1 = filesha1("${path.module}/files/register-server.sh")
+    # Refresh resource when lb fqdn changes
+    lb_id = local.instance_fqdns[count.index]
+  }
+
+  count = var.lb_count
+
+  provisioner "local-exec" {
+    command = "${path.module}/files/register-server.sh"
+    environment = {
+      CONTROL_VSHN_NET_TOKEN = var.control_vshn_net_token
+      SERVER_FQDN            = local.instance_fqdns[count.index]
+      SERVER_REGION          = "${var.region}.ch"
+      # Cluster id is used as encdata stage
+      CLUSTER_ID = var.cluster_id
+    }
+  }
+}
+
+resource "gitfile_checkout" "appuio_hieradata" {
+  repo = "https://${var.hieradata_repo_user}@git.vshn.net/appuio/appuio_hieradata.git"
+  path = "${path.root}/appuio_hieradata"
+
+  count = var.lb_count > 0 ? 1 : 0
+
+  lifecycle {
+    ignore_changes = [
+      branch
+    ]
+  }
+}
+
+resource "local_file" "lb_hieradata" {
+  count = var.lb_count > 0 ? 1 : 0
+
+  content = templatefile(
+    "${path.module}/templates/hieradata.yaml.tmpl",
+    {
+      "cluster_id"   = var.cluster_id
+      "api_vip"      = cidrhost(cloudscale_floating_ip.api_vip[0].network, 0)
+      "router_vip"   = cidrhost(cloudscale_floating_ip.router_vip[0].network, 0)
+      "api_secret"   = var.lb_cloudscale_api_secret
+      "internal_vip" = cidrhost(var.privnet_cidr, 100)
+      "nat_vip"      = cidrhost(cloudscale_floating_ip.nat_vip[0].network, 0)
+      "nodes"        = local.instance_fqdns
+      "backends" = {
+        "api"    = formatlist("etcd-%d.${local.node_name_suffix}", range(3))
+        "router" = module.infra.ip_addresses[*],
+      }
+      "bootstrap_node" = var.bootstrap_count > 0 ? cidrhost(var.privnet_cidr, 10) : ""
+  })
+
+  filename             = "${path.cwd}/appuio_hieradata/lbaas/${var.cluster_id}.yaml"
+  file_permission      = "0644"
+  directory_permission = "0755"
+
+  depends_on = [
+    gitfile_checkout.appuio_hieradata[0]
+  ]
+
+  provisioner "local-exec" {
+    command = "${path.module}/files/commit-hieradata.sh ${var.cluster_id}"
+  }
+}
+
+data "local_file" "hieradata_mr_url" {
+  filename = "${path.cwd}/.mr_url.txt"
+
+  depends_on = [
+    local_file.lb_hieradata
+  ]
+}
+
 resource "cloudscale_server" "lb" {
-  count            = var.lb_count
-  name             = "${random_id.lb[count.index].hex}.${local.node_name_suffix}"
-  zone_slug        = "${var.region}1"
-  flavor_slug      = "plus-8"
-  image_slug       = "ubuntu-20.04"
-  server_group_ids = var.lb_count != 0 ? [cloudscale_server_group.lb[0].id] : []
-  volume_size_gb   = 50
-  ssh_keys         = var.ssh_keys
+  count                          = var.lb_count
+  name                           = local.instance_fqdns[count.index]
+  zone_slug                      = "${var.region}1"
+  flavor_slug                    = "plus-8"
+  image_slug                     = "ubuntu-20.04"
+  server_group_ids               = var.lb_count != 0 ? [cloudscale_server_group.lb[0].id] : []
+  volume_size_gb                 = 50
+  ssh_keys                       = var.ssh_keys
+  skip_waiting_for_ssh_host_keys = true
   interfaces {
     type = "public"
   }
   interfaces {
     type         = "private"
     network_uuid = cloudscale_network.privnet.id
-    no_address   = true
   }
   lifecycle {
     create_before_destroy = true
   }
-  user_data = <<-EOF
-    #cloud-config
-    package_update: true
-    packages:
-    - haproxy
-    - keepalived
-    bootcmd:
-    - "iptables -t nat -A POSTROUTING -o ens3 -j MASQUERADE"
-    - "sysctl -w net.ipv4.ip_forward=1"
-    - "sysctl -w net.ipv4.ip_nonlocal_bind=1"
-    - "ip link set ens7 up"
-    - "ip address add ${cidrhost(var.privnet_cidr, 2 + count.index)}/24 dev ens7"
-    write_files:
-    - path: "/etc/keepalived/keepalived.conf"
-      encoding: b64
-      content: ${base64encode(templatefile("${path.module}/templates/keepalived.conf", {
-  "api_eip" = random_id.lb[count.index].keepers.api_eip
-  "api_int" = cidrhost(var.privnet_cidr, 100)
-  "gateway" = cloudscale_subnet.privnet_subnet.gateway_address
-  "prio"    = "${(var.lb_count - count.index) * 10}"
-  }))}
-    - path: "/etc/haproxy/haproxy.cfg"
-      encoding: b64
-      content: ${base64encode(templatefile("${path.module}/templates/haproxy.cfg", {
-  "api_eip" = cidrhost(cloudscale_floating_ip.api_vip[0].network, 0)
-  "api_int" = cidrhost(var.privnet_cidr, 100)
-  "api_servers" = [
-    cidrhost(var.privnet_cidr, 10),
-    "etcd-0.${local.node_name_suffix}",
-    "etcd-1.${local.node_name_suffix}",
-    "etcd-2.${local.node_name_suffix}",
-  ]
-  "infra_servers" = length(random_id.lb[count.index].keepers.infra_servers) > 0 ? split(",", random_id.lb[count.index].keepers.infra_servers) : []
-}))}
-    EOF
-}
 
-resource "null_resource" "api_vip_assignement" {
-  count = var.lb_count != 0 ? 1 : 0
-  triggers = {
-    api_eip = cloudscale_floating_ip.api_vip[0].network
-    server  = cloudscale_server.lb[0].id
-  }
+  user_data = format("#cloud-config\n%s", yamlencode(merge(
+    local.common_user_data,
+    {
+      "fqdn"     = local.instance_fqdns[count.index],
+      "hostname" = random_id.lb[count.index].hex,
+    }
+  )))
 
-  provisioner "local-exec" {
-    command = <<-EOF
-      wget --header "Authorization: Bearer $CLOUDSCALE_TOKEN" \
-        -O - \
-        --post-data server=${cloudscale_server.lb[0].id} \
-        https://api.cloudscale.ch/v1/floating-ips/${cidrhost(cloudscale_floating_ip.api_vip[0].network, 0)}
-      EOF
-  }
+  depends_on = [
+    null_resource.register_lb,
+    local_file.lb_hieradata[0]
+  ]
 }

main.tf

@@ -27,3 +27,31 @@ resource "cloudscale_floating_ip" "api_vip" {
     ]
   }
 }
+
+resource "cloudscale_floating_ip" "router_vip" {
+  count       = var.lb_count != 0 ? 1 : 0
+  ip_version  = 4
+  region_slug = var.region
+
+  lifecycle {
+    ignore_changes = [
+      # Will be handled by Keepalived (Ursula)
+      server,
+      next_hop,
+    ]
+  }
+}
+
+resource "cloudscale_floating_ip" "nat_vip" {
+  count       = var.lb_count != 0 ? 1 : 0
+  ip_version  = 4
+  region_slug = var.region
+
+  lifecycle {
+    ignore_changes = [
+      # Will be handled by Keepalived (Ursula)
+      server,
+      next_hop,
+    ]
+  }
+}

modules/node-group/versions.tf → modules/node-group/providers.tf

@@ -1,11 +1,11 @@
 terraform {
+  required_version = ">= 0.14"
   required_providers {
     cloudscale = {
-      source = "terraform-providers/cloudscale"
+      source = "cloudscale-ch/cloudscale"
     }
     random = {
       source = "hashicorp/random"
     }
   }
-  required_version = ">= 0.13"
 }

outputs.tf

@@ -1,10 +1,13 @@
 output "dns_entries" {
   value = templatefile("${path.module}/templates/dns.zone", {
     "node_name_suffix" = local.node_name_suffix,
-    "eip_api"          = var.lb_count != 0 ? split("/", cloudscale_floating_ip.api_vip[0].network)[0] : ""
-    "api_int"          = cidrhost(var.privnet_cidr, 100),
+    "api_vip"          = var.lb_count != 0 ? split("/", cloudscale_floating_ip.api_vip[0].network)[0] : ""
+    "router_vip"       = var.lb_count != 0 ? split("/", cloudscale_floating_ip.router_vip[0].network)[0] : ""
+    "internal_vip"     = cidrhost(var.privnet_cidr, 100),
     "masters"          = module.master.ip_addresses,
     "cluster_id"       = var.cluster_id,
+    "lbs"              = cloudscale_server.lb[*].public_ipv4_address,
+    "lb_hostnames"     = random_id.lb[*].hex
   })
 }
 
@@ -31,3 +34,7 @@ output "ignition_ca" {
 output "api_int" {
   value = "api-int.${local.node_name_suffix}"
 }
+
+output "hieradata_mr" {
+  value = data.local_file.hieradata_mr_url.content
+}