diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000..f613d83 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,19 @@ +name: Pull Request +on: + pull_request: + branches: + - master + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Extract Terraform version from constraints in module + run: echo TF_VERSION=$(grep "^[[:space:]]\+required_version = \"" provider.tf | cut -d= -f2- | tr -d ' "') >> $GITHUB_ENV + - uses: hashicorp/setup-terraform@v1 + with: + terraform_version: ${{ env.TF_VERSION }} + - run: terraform fmt -check -recursive + - run: terraform init -input=false + - run: terraform validate diff --git a/files/commit-hieradata.sh b/files/commit-hieradata.sh new file mode 100755 index 0000000..f354b5c --- /dev/null +++ b/files/commit-hieradata.sh @@ -0,0 +1,75 @@ +#!/bin/sh + +readonly cluster_id=$1 +readonly branch="tf/lbaas/${cluster_id}" + +cd appuio_hieradata || exit 1 + +git config user.name "${GIT_AUTHOR_NAME}" +git config user.email "${GIT_AUTHOR_EMAIL}" + +# Checkout feature branch +# 1. try to check out as tracking branch from origin +# 2. checkout as new branch +# 3. checkout existing local branch +# For existing local branch, amend existing commit +if ! git checkout -t origin/"${branch}"; then + if ! git checkout -b "${branch}"; then + git checkout "${branch}" + fi +fi + +git add "lbaas/${cluster_id}.yaml" + +status=$(git status --porcelain) +echo "'${status}'" + +commit_message="Update LBaaS hieradata for ${cluster_id}" +push=1 +if [ "${status}" = "M lbaas/${cluster_id}.yaml" ]; then + git commit -m"${commit_message}" +elif [ "${status}" != "" ]; then + # assume new hieradata + commit_message="Create LBaaS hieradata for ${cluster_id}" + git commit -m "${commit_message}" +else + push=0 +fi + +if [ "${push}" -eq 1 ]; then + # Push branch to origin and set upstream + git push origin "${branch}" +fi + +# Always set branch's upstream to origin/master. +# +# If we would set the branch's upstream to the pushed branch, subsequent +# terraform runs break if the upstream branch has been merged or deleted. +# +# If we only set the upstream when pushing, subsequent terraform runs break if +# there's been no changes in the hieradata. +git branch -u origin/master + +# Create MR if none exists yet +open_mrs=$(curl -sH "Authorization: Bearer ${HIERADATA_REPO_TOKEN}" \ + "https://git.vshn.net/api/v4/projects/368/merge_requests?state=opened&source_branch=tf/lbaas/${cluster_id}") +if [ "${push}" -eq 0 ]; then + mr_url="No changes, skipping push and MR creation" +elif [ "${open_mrs}" = "[]" ]; then + # create MR + mr_url=$(curl -XPOST -sH "Authorization: Bearer ${HIERADATA_REPO_TOKEN}" \ + -H"Content-type: application/json" \ + "https://git.vshn.net/api/v4/projects/368/merge_requests" \ + -d \ + "{ + \"id\": 368, + \"source_branch\": \"tf/lbaas/${cluster_id}\", + \"target_branch\": \"master\", + \"title\": \"${commit_message}\", + \"remove_source_branch\": true + }" | jq -r '.web_url') +else + mr_url=$(echo "${open_mrs}" | jq -r '.[0].web_url') +fi + +echo "${mr_url}" > /tf/.mr_url.txt diff --git a/files/register-server.sh b/files/register-server.sh new file mode 100755 index 0000000..7a109ff --- /dev/null +++ b/files/register-server.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +set -eo pipefail + +curl -s -X POST -H "X-AccessToken: ${CONTROL_VSHN_NET_TOKEN}" \ + https://control.vshn.net/api/servers/1/appuio/ \ + -d "{ + \"customer\": \"appuio\", + \"fqdn\": \"${SERVER_FQDN}\", + \"location\": \"cloudscale\", + \"region\": \"${SERVER_REGION}\", + \"environment\": \"AppuioLbaas\", + \"project\": \"lbaas\", + \"role\": \"lb\", + \"stage\": \"${CLUSTER_ID}\" + }" diff --git a/lb.tf b/lb.tf index fdfc9f8..88e4120 100644 --- a/lb.tf +++ b/lb.tf @@ -15,76 +15,154 @@ resource "cloudscale_server_group" "lb" { zone_slug = "${var.region}1" } +locals { + instance_fqdns = formatlist("%s.${local.node_name_suffix}", random_id.lb[*].hex) + + common_user_data = { + "package_update" = true, + "package_upgrade" = true, + "runcmd" = [ + "sleep '5'", + "wget -O /tmp/puppet-source.deb https://apt.puppetlabs.com/puppet6-release-focal.deb", + "dpkg -i /tmp/puppet-source.deb", + "rm /tmp/puppet-source.deb", + "apt-get update", + "apt-get -y install puppet-agent", + "apt-get -y purge snapd", + "mkdir -p /etc/puppetlabs/facter/facts.d", + "netplan apply", + ["bash", "-c", + "set +e -x; for ((i=0; i < 3; ++i)); do /opt/puppetlabs/bin/puppet facts && break; done; for ((i=0; i < 3; ++i)); do /opt/puppetlabs/bin/puppet agent -t --server master.puppet.vshn.net && break; done"], + "sleep 5", + "shutdown --reboot +1 'Reboot for system setup'", + ], + "manage_etc_hosts" = true, + "write_files" = [ + { + path = "/etc/netplan/60-ens4.yaml" + "encoding" = "b64" + "content" = base64encode(yamlencode({ + "network" = { + "ethernets" = { + "ens4" = { + "dhcp4" = true, + }, + }, + "version" = 2, + } + })) + } + ] + } +} + +resource "null_resource" "register_lb" { + triggers = { + # Refresh resource when script changes -- this is probaby not required for production + script_sha1 = filesha1("${path.module}/files/register-server.sh") + # Refresh resource when lb fqdn changes + lb_id = local.instance_fqdns[count.index] + } + + count = var.lb_count + + provisioner "local-exec" { + command = "${path.module}/files/register-server.sh" + environment = { + CONTROL_VSHN_NET_TOKEN = var.control_vshn_net_token + SERVER_FQDN = local.instance_fqdns[count.index] + SERVER_REGION = "${var.region}.ch" + # Cluster id is used as encdata stage + CLUSTER_ID = var.cluster_id + } + } +} + +resource "gitfile_checkout" "appuio_hieradata" { + repo = "https://${var.hieradata_repo_user}@git.vshn.net/appuio/appuio_hieradata.git" + path = "${path.root}/appuio_hieradata" + + count = var.lb_count > 0 ? 1 : 0 + + lifecycle { + ignore_changes = [ + branch + ] + } +} + +resource "local_file" "lb_hieradata" { + count = var.lb_count > 0 ? 1 : 0 + + content = templatefile( + "${path.module}/templates/hieradata.yaml.tmpl", + { + "cluster_id" = var.cluster_id + "api_vip" = cidrhost(cloudscale_floating_ip.api_vip[0].network, 0) + "router_vip" = cidrhost(cloudscale_floating_ip.router_vip[0].network, 0) + "api_secret" = var.lb_cloudscale_api_secret + "internal_vip" = cidrhost(var.privnet_cidr, 100) + "nat_vip" = cidrhost(cloudscale_floating_ip.nat_vip[0].network, 0) + "nodes" = local.instance_fqdns + "backends" = { + "api" = formatlist("etcd-%d.${local.node_name_suffix}", range(3)) + "router" = module.infra.ip_addresses[*], + } + "bootstrap_node" = var.bootstrap_count > 0 ? cidrhost(var.privnet_cidr, 10) : "" + }) + + filename = "${path.cwd}/appuio_hieradata/lbaas/${var.cluster_id}.yaml" + file_permission = "0644" + directory_permission = "0755" + + depends_on = [ + gitfile_checkout.appuio_hieradata[0] + ] + + provisioner "local-exec" { + command = "${path.module}/files/commit-hieradata.sh ${var.cluster_id}" + } +} + +data "local_file" "hieradata_mr_url" { + filename = "${path.cwd}/.mr_url.txt" + + depends_on = [ + local_file.lb_hieradata + ] +} + resource "cloudscale_server" "lb" { - count = var.lb_count - name = "${random_id.lb[count.index].hex}.${local.node_name_suffix}" - zone_slug = "${var.region}1" - flavor_slug = "plus-8" - image_slug = "ubuntu-20.04" - server_group_ids = var.lb_count != 0 ? [cloudscale_server_group.lb[0].id] : [] - volume_size_gb = 50 - ssh_keys = var.ssh_keys + count = var.lb_count + name = local.instance_fqdns[count.index] + zone_slug = "${var.region}1" + flavor_slug = "plus-8" + image_slug = "ubuntu-20.04" + server_group_ids = var.lb_count != 0 ? [cloudscale_server_group.lb[0].id] : [] + volume_size_gb = 50 + ssh_keys = var.ssh_keys + skip_waiting_for_ssh_host_keys = true interfaces { type = "public" } interfaces { type = "private" network_uuid = cloudscale_network.privnet.id - no_address = true } lifecycle { create_before_destroy = true } - user_data = <<-EOF - #cloud-config - package_update: true - packages: - - haproxy - - keepalived - bootcmd: - - "iptables -t nat -A POSTROUTING -o ens3 -j MASQUERADE" - - "sysctl -w net.ipv4.ip_forward=1" - - "sysctl -w net.ipv4.ip_nonlocal_bind=1" - - "ip link set ens7 up" - - "ip address add ${cidrhost(var.privnet_cidr, 2 + count.index)}/24 dev ens7" - write_files: - - path: "/etc/keepalived/keepalived.conf" - encoding: b64 - content: ${base64encode(templatefile("${path.module}/templates/keepalived.conf", { - "api_eip" = random_id.lb[count.index].keepers.api_eip - "api_int" = cidrhost(var.privnet_cidr, 100) - "gateway" = cloudscale_subnet.privnet_subnet.gateway_address - "prio" = "${(var.lb_count - count.index) * 10}" - }))} - - path: "/etc/haproxy/haproxy.cfg" - encoding: b64 - content: ${base64encode(templatefile("${path.module}/templates/haproxy.cfg", { - "api_eip" = cidrhost(cloudscale_floating_ip.api_vip[0].network, 0) - "api_int" = cidrhost(var.privnet_cidr, 100) - "api_servers" = [ - cidrhost(var.privnet_cidr, 10), - "etcd-0.${local.node_name_suffix}", - "etcd-1.${local.node_name_suffix}", - "etcd-2.${local.node_name_suffix}", - ] - "infra_servers" = length(random_id.lb[count.index].keepers.infra_servers) > 0 ? split(",", random_id.lb[count.index].keepers.infra_servers) : [] -}))} - EOF -} -resource "null_resource" "api_vip_assignement" { - count = var.lb_count != 0 ? 1 : 0 - triggers = { - api_eip = cloudscale_floating_ip.api_vip[0].network - server = cloudscale_server.lb[0].id - } + user_data = format("#cloud-config\n%s", yamlencode(merge( + local.common_user_data, + { + "fqdn" = local.instance_fqdns[count.index], + "hostname" = random_id.lb[count.index].hex, + } + ))) - provisioner "local-exec" { - command = <<-EOF - wget --header "Authorization: Bearer $CLOUDSCALE_TOKEN" \ - -O - \ - --post-data server=${cloudscale_server.lb[0].id} \ - https://api.cloudscale.ch/v1/floating-ips/${cidrhost(cloudscale_floating_ip.api_vip[0].network, 0)} - EOF - } + depends_on = [ + null_resource.register_lb, + local_file.lb_hieradata[0] + ] } diff --git a/main.tf b/main.tf index 97071a3..1a2d79c 100644 --- a/main.tf +++ b/main.tf @@ -27,3 +27,31 @@ resource "cloudscale_floating_ip" "api_vip" { ] } } + +resource "cloudscale_floating_ip" "router_vip" { + count = var.lb_count != 0 ? 1 : 0 + ip_version = 4 + region_slug = var.region + + lifecycle { + ignore_changes = [ + # Will be handled by Keepalived (Ursula) + server, + next_hop, + ] + } +} + +resource "cloudscale_floating_ip" "nat_vip" { + count = var.lb_count != 0 ? 1 : 0 + ip_version = 4 + region_slug = var.region + + lifecycle { + ignore_changes = [ + # Will be handled by Keepalived (Ursula) + server, + next_hop, + ] + } +} diff --git a/modules/node-group/versions.tf b/modules/node-group/providers.tf similarity index 60% rename from modules/node-group/versions.tf rename to modules/node-group/providers.tf index 25523b4..9bc20a7 100644 --- a/modules/node-group/versions.tf +++ b/modules/node-group/providers.tf @@ -1,11 +1,11 @@ terraform { + required_version = ">= 0.14" required_providers { cloudscale = { - source = "terraform-providers/cloudscale" + source = "cloudscale-ch/cloudscale" } random = { source = "hashicorp/random" } } - required_version = ">= 0.13" } diff --git a/outputs.tf b/outputs.tf index 5e4bd91..66031c0 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,10 +1,13 @@ output "dns_entries" { value = templatefile("${path.module}/templates/dns.zone", { "node_name_suffix" = local.node_name_suffix, - "eip_api" = var.lb_count != 0 ? split("/", cloudscale_floating_ip.api_vip[0].network)[0] : "" - "api_int" = cidrhost(var.privnet_cidr, 100), + "api_vip" = var.lb_count != 0 ? split("/", cloudscale_floating_ip.api_vip[0].network)[0] : "" + "router_vip" = var.lb_count != 0 ? split("/", cloudscale_floating_ip.router_vip[0].network)[0] : "" + "internal_vip" = cidrhost(var.privnet_cidr, 100), "masters" = module.master.ip_addresses, "cluster_id" = var.cluster_id, + "lbs" = cloudscale_server.lb[*].public_ipv4_address, + "lb_hostnames" = random_id.lb[*].hex }) } @@ -31,3 +34,7 @@ output "ignition_ca" { output "api_int" { value = "api-int.${local.node_name_suffix}" } + +output "hieradata_mr" { + value = data.local_file.hieradata_mr_url.content +} diff --git a/versions.tf b/provider.tf similarity index 66% rename from versions.tf rename to provider.tf index fce7092..fe59931 100644 --- a/versions.tf +++ b/provider.tf @@ -1,8 +1,9 @@ terraform { + required_version = ">= 0.14" required_providers { cloudscale = { source = "cloudscale-ch/cloudscale" - version = ">= 2.3" + version = ">= 3.0" } null = { source = "hashicorp/null" @@ -12,6 +13,9 @@ terraform { source = "hashicorp/random" version = ">= 2.3" } + gitfile = { + source = "igal-s/gitfile" + version = "1.0.0" + } } - required_version = ">= 0.13" } diff --git a/templates/dns.zone b/templates/dns.zone index 81a8d1d..6f1cd64 100644 --- a/templates/dns.zone +++ b/templates/dns.zone @@ -1,9 +1,13 @@ ; Cluster ${cluster_id} $ORIGIN ${node_name_suffix}. -api IN A ${eip_api} -api-int IN A ${api_int} -*.apps IN CNAME api +api IN A ${api_vip} +api-int IN A ${internal_vip} +*.apps IN A ${router_vip} + +%{ for i, addr in lbs ~} +${lb_hostnames[i]} IN A ${addr} +%{ endfor ~} %{ for i, addr in masters ~} etcd-${i} IN A ${addr} diff --git a/templates/haproxy.cfg b/templates/haproxy.cfg deleted file mode 100644 index 3005835..0000000 --- a/templates/haproxy.cfg +++ /dev/null @@ -1,66 +0,0 @@ -global - daemon - log /dev/log local0 - log /dev/log local1 notice - maxconn 2048 - pidfile /var/run/haproxy.pid - tune.ssl.default-dh-param 2048 - -defaults - log global - mode tcp - option tcplog - option httpchk GET /healthz - balance roundrobin - timeout connect 5s - timeout client 50s - timeout tunnel 1h - timeout http-request 10s - timeout server 61m - default-server init-addr last,libc,none - -resolvers localdns - nameserver dns1 127.0.0.53:53 - accepted_payload_size 8192 # allow larger DNS payloads - -frontend fe-api - bind ${api_int}:6443 - default_backend be-api - -frontend fe-api-ext - bind ${api_eip}:6443 - default_backend be-api - -frontend fe-ign - bind ${api_int}:22623 - default_backend be-ign - -frontend fe-router-http - bind ${api_eip}:80 - default_backend be-router-http - -frontend fe-router-https - bind ${api_eip}:443 - default_backend be-router-https - -backend be-api -%{ for i, addr in api_servers ~} - server master-${i} ${addr}:6443 check check-ssl verify none resolvers localdns -%{ endfor ~} - -backend be-ign -%{ for i, addr in api_servers ~} - server master-${i} ${addr}:22623 check check-ssl verify none resolvers localdns -%{ endfor ~} - -backend be-router-http - option httpchk GET /healthz/ready -%{ for i, addr in infra_servers ~} - server infra-${i} ${addr}:80 check port 1936 check-ssl verify none resolvers localdns -%{ endfor ~} - -backend be-router-https - option httpchk GET /healthz/ready -%{ for i, addr in infra_servers ~} - server infra-${i} ${addr}:443 check port 1936 check-ssl verify none resolvers localdns -%{ endfor ~} diff --git a/templates/hieradata.yaml.tmpl b/templates/hieradata.yaml.tmpl new file mode 100644 index 0000000..2189c54 --- /dev/null +++ b/templates/hieradata.yaml.tmpl @@ -0,0 +1,28 @@ +# Managed by Terraform for Project Syn cluster ${cluster_id} +profile_openshift4_gateway::nodes: +%{ for node in nodes ~} + - ${node} +%{ endfor ~} +profile_openshift4_gateway::public_interface: ens3 +profile_openshift4_gateway::private_interfaces: + - ens4 +profile_openshift4_gateway::floating_addresses: + api: ${api_vip} + nat: ${nat_vip} + router: ${router_vip} +profile_openshift4_gateway::floating_address_provider: cloudscale +profile_openshift4_gateway::internal_vip: ${internal_vip} +profile_openshift4_gateway::floating_address_settings: + token: ${api_secret} +profile_openshift4_gateway::backends: + 'api':%{ if length(backends["api"]) == 0 && bootstrap_node == "" } []%{ endif } +%{ for be in backends["api"] ~} + - ${be} +%{ endfor ~} +%{ if bootstrap_node != "" ~} + - ${bootstrap_node} +%{ endif ~} + 'router':%{ if length(backends["router"]) == 0 } []%{ endif } +%{ for be in backends["router"] ~} + - ${be} +%{ endfor ~} diff --git a/templates/keepalived.conf b/templates/keepalived.conf deleted file mode 100644 index 389333b..0000000 --- a/templates/keepalived.conf +++ /dev/null @@ -1,40 +0,0 @@ -vrrp_sync_group VG1 { - group { - EXT - INT - } -} - -vrrp_instance EXT { - state MASTER - interface ens3 - virtual_router_id 50 - priority ${prio} - advert_int 1 - authentication { - auth_type PASS - auth_pass passw123 - } - virtual_ipaddress { - # EIP for API - ${api_eip} - } -} - -vrrp_instance INT { - state MASTER - interface ens7 - virtual_router_id 51 - priority ${prio} - advert_int 1 - authentication { - auth_type PASS - auth_pass passw456 - } - virtual_ipaddress { - # VIP for internal API - ${api_int} - # Gateway IP for the private network - ${gateway} - } -} diff --git a/variables.tf b/variables.tf index f463547..47e2bc8 100644 --- a/variables.tf +++ b/variables.tf @@ -91,3 +91,15 @@ variable "image_slug" { description = "Image to use for nodes" default = "custom:rhcos-4.7" } + +variable "lb_cloudscale_api_secret" { + type = string +} + +variable "hieradata_repo_user" { + type = string +} + +variable "control_vshn_net_token" { + type = string +}