Install nginx ingress controller instead of Traefik

garutilorenzo · Aug 9, 2022 · ccf55bf · ccf55bf
1 parent 73d4f90
commit ccf55bf
Show file tree

Hide file tree

Showing 6 changed files with 226 additions and 23 deletions.
diff --git a/README.md b/README.md
@@ -43,6 +43,10 @@ edit the main.tf files and set the following variables:
 | `default_instance_profile_name`  | `no`  | Instance profile name. Default: AWSEC2K3SInstanceProfile |
 | `default_iam_role`  | `no`  | IAM role name. Default: AWSEC2K3SRole |
 | `create_extlb`  | `no`  | Boolean value true/false, specify true for deploy an external LB pointing to k3s worker nodes. Default: false |
+| `install_nginx_ingress`  | `no`  | Boolean value, install kubernetes nginx ingress controller instead of Traefik. Default: true. For more information see [Nginx ingress controller](#nginx-ingress-controller) |
+| `install_certmanager`  | `no`  | Boolean value, install [cert manager](https://cert-manager.io/) "Cloud native certificate management". Default: true  |
+| `certmanager_release`  | `no`  | Cert manager release. Default: v1.8.2  |
+| `certmanager_email_address`  | `no`  | Email address used for signing https certificates. Defaul: changeme@example.com  |
 | `extlb_http_port`  | `no`  | http port used by the external LB. Default: 80 |
 | `extlb_https_port`  | `no`  | https port used by the external LB. Default: 443  |
 | `PATH_TO_PUBLIC_KEY`     | `no`       | Path to your public ssh key (Default: "~/.ssh/id_rsa.pub) |
@@ -135,6 +139,62 @@ Notes about the auoscaling group:
 
 You can change this setting by editing the value of on_demand_percentage_above_base_capacity in asg.tf. You can require that all the EC2 will be launced using on-demand instances setting on_demand_percentage_above_base_capacity to 100. More details [here](https://docs.aws.amazon.com/autoscaling/ec2/APIReference/API_InstancesDistribution.html)
 
+### Nginx ingress controller
+
+In this environment [Nginx ingress controller](https://kubernetes.github.io/ingress-nginx/) is used instead of the standard [Traefik](https://traefik.io/) ingress controller.
+
+The installation is the [bare metal](https://kubernetes.github.io/ingress-nginx/deploy/#bare-metal-clusters) installation, the ingress controller then is exposed via K3s LoadBalancer Service.
+
+```yaml
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: ingress-nginx-controller-loadbalancer
+  namespace: ingress-nginx
+spec:
+  selector:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+  ports:
+    - name: http
+      port: 80
+      protocol: TCP
+      targetPort: 80
+    - name: https
+      port: 443
+      protocol: TCP
+      targetPort: 443
+  type: LoadBalancer
+```
+
+To get the real ip address of the clients using a public L4 load balancer we need to use the proxy protocol feature of nginx ingress controller:
+
+```yaml
+---
+apiVersion: v1
+data:
+  allow-snippet-annotations: "true"
+  enable-real-ip: "true"
+  proxy-real-ip-cidr: "0.0.0.0/0"
+  proxy-body-size: "20m"
+  use-proxy-protocol: "true"
+kind: ConfigMap
+metadata:
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    helm.sh/chart: ingress-nginx-4.0.16
+  name: ingress-nginx-controller
+  namespace: ingress-nginx
+```
+
+and enable the proxy protocol on the load balancer target group, *proxy_protocol_v2* set to true.
 
 ## Instances used
 

diff --git a/k3s_cluster/data.tf b/k3s_cluster/data.tf
@@ -45,7 +45,16 @@ data "template_cloudinit_config" "k3s_server" {
 
   part {
     content_type = "text/x-shellscript"
-    content      = templatefile("${path.module}/files/k3s-install-server.sh", { k3s_token = var.k3s_token, is_k3s_server = true, k3s_url = aws_lb.k3s-server-lb.dns_name, k3s_tls_san = aws_lb.k3s-server-lb.dns_name })
+    content = templatefile("${path.module}/files/k3s-install-server.sh", {
+      k3s_token                 = var.k3s_token,
+      is_k3s_server             = true,
+      install_nginx_ingress     = var.install_nginx_ingress,
+      install_certmanager       = var.install_certmanager,
+      certmanager_release       = var.certmanager_release,
+      certmanager_email_address = var.certmanager_email_address,
+      k3s_url                   = aws_lb.k3s-server-lb.dns_name,
+      k3s_tls_san               = aws_lb.k3s-server-lb.dns_name
+    })
   }
 }
 
@@ -62,6 +71,11 @@ data "template_cloudinit_config" "k3s_agent" {
 
   part {
     content_type = "text/x-shellscript"
-    content      = templatefile("${path.module}/files/k3s-install-agent.sh", { k3s_token = var.k3s_token, is_k3s_server = false, k3s_url = aws_lb.k3s-server-lb.dns_name, k3s_tls_san = aws_lb.k3s-server-lb.dns_name })
+    content = templatefile("${path.module}/files/k3s-install-agent.sh", {
+      k3s_token     = var.k3s_token,
+      is_k3s_server = false,
+      k3s_url       = aws_lb.k3s-server-lb.dns_name,
+      k3s_tls_san   = aws_lb.k3s-server-lb.dns_name
+    })
   }
 }
diff --git a/k3s_cluster/extlb.tf b/k3s_cluster/extlb.tf
@@ -34,11 +34,11 @@ resource "aws_lb_listener" "external-lb-listener-http" {
 }
 
 resource "aws_lb_target_group" "external-lb-tg-http" {
-  count    = var.create_extlb ? 1 : 0
-  port     = var.extlb_http_port
-  protocol = "TCP"
-  vpc_id   = var.vpc_id
-
+  count             = var.create_extlb ? 1 : 0
+  port              = var.extlb_http_port
+  protocol          = "TCP"
+  vpc_id            = var.vpc_id
+  proxy_protocol_v2 = true
 
   depends_on = [
     aws_lb.external-lb
@@ -90,11 +90,11 @@ resource "aws_lb_listener" "external-lb-listener-https" {
 }
 
 resource "aws_lb_target_group" "external-lb-tg-https" {
-  count    = var.create_extlb ? 1 : 0
-  port     = var.extlb_https_port
-  protocol = "TCP"
-  vpc_id   = var.vpc_id
-
+  count             = var.create_extlb ? 1 : 0
+  port              = var.extlb_https_port
+  protocol          = "TCP"
+  vpc_id            = var.vpc_id
+  proxy_protocol_v2 = true
 
   depends_on = [
     aws_lb.external-lb

diff --git a/k3s_cluster/files/k3s-install-server.sh b/k3s_cluster/files/k3s-install-server.sh
@@ -16,16 +16,20 @@ first_instance=$(aws ec2 describe-instances --filters Name=tag-value,Values=k3s-
 instance_id=$(curl -s http://169.254.169.254/latest/meta-data/instance-id)
 first_last="last"
 
+%{ if install_nginx_ingress } 
+disable_traefik="--disable traefik"
+%{ endif }
+
 if [[ "$first_instance" == "$instance_id" ]]; then
     echo "I'm the first yeeee: Cluster init!"
     first_last="first"
-    until (curl -sfL https://get.k3s.io | K3S_TOKEN=${k3s_token} sh -s - --cluster-init --node-ip $local_ip --advertise-address $local_ip --flannel-iface $flannel_iface --tls-san ${k3s_tls_san} --kubelet-arg="provider-id=aws:///$provider_id"); do
+    until (curl -sfL https://get.k3s.io | K3S_TOKEN=${k3s_token} sh -s - --cluster-init $disable_traefik --node-ip $local_ip --advertise-address $local_ip --flannel-iface $flannel_iface --tls-san ${k3s_tls_san} --kubelet-arg="provider-id=aws:///$provider_id"); do
       echo 'k3s did not install correctly'
       sleep 2
     done
 else
     echo "I'm like England :( Cluster join"
-    until (curl -sfL https://get.k3s.io | K3S_TOKEN=${k3s_token} sh -s - --server https://${k3s_url}:6443 --node-ip $local_ip --advertise-address $local_ip --flannel-iface $flannel_iface --tls-san ${k3s_tls_san} --kubelet-arg="provider-id=aws:///$provider_id"  ); do
+    until (curl -sfL https://get.k3s.io | K3S_TOKEN=${k3s_token} sh -s - --server https://${k3s_url}:6443 $disable_traefik --node-ip $local_ip --advertise-address $local_ip --flannel-iface $flannel_iface --tls-san ${k3s_tls_san} --kubelet-arg="provider-id=aws:///$provider_id"  ); do
       echo 'k3s did not install correctly'
       sleep 2
     done
@@ -42,6 +46,109 @@ if [[ "$first_last" == "first" ]]; then
   echo 'Install node termination handler'
   kubectl apply -f https://github.com/aws/aws-node-termination-handler/releases/download/v1.13.3/all-resources.yaml
 fi
+
+%{ if install_nginx_ingress }
+if [[ "$first_last" == "first" ]]; then
+    kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.1.1/deploy/static/provider/baremetal/deploy.yaml
+cat << 'EOF' > /root/nginx-ingress-resources.yaml
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: ingress-nginx-controller-loadbalancer
+  namespace: ingress-nginx
+spec:
+  selector:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+  ports:
+    - name: http
+      port: 80
+      protocol: TCP
+      targetPort: 80
+    - name: https
+      port: 443
+      protocol: TCP
+      targetPort: 443
+  type: LoadBalancer
+---
+apiVersion: v1
+data:
+  allow-snippet-annotations: "true"
+  enable-real-ip: "true"
+  proxy-real-ip-cidr: "0.0.0.0/0"
+  proxy-body-size: "20m"
+  use-proxy-protocol: "true"
+kind: ConfigMap
+metadata:
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    helm.sh/chart: ingress-nginx-4.0.16
+  name: ingress-nginx-controller
+  namespace: ingress-nginx
+EOF
+    kubectl apply -f /root/nginx-ingress-resources.yaml
+fi
+%{ endif }
+
+%{ if install_certmanager }
+if [[ "$first_last" == "first" ]]; then
+    kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/${certmanager_release}/cert-manager.yaml
+
+cat << 'EOF' > /root/staging_issuer.yaml
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+ name: letsencrypt-staging
+ namespace: cert-manager
+spec:
+ acme:
+   # The ACME server URL
+   server: https://acme-staging-v02.api.letsencrypt.org/directory
+   # Email address used for ACME registration
+   email: ${certmanager_email_address}
+   # Name of a secret used to store the ACME account private key
+   privateKeySecretRef:
+     name: letsencrypt-staging
+   # Enable the HTTP-01 challenge provider
+   solvers:
+   - http01:
+       ingress:
+         class:  nginx
+EOF
+
+cat << 'EOF' > /root/prod_issuer.yaml
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: letsencrypt-prod
+  namespace: cert-manager
+spec:
+  acme:
+    # The ACME server URL
+    server: https://acme-v02.api.letsencrypt.org/directory
+    # Email address used for ACME registration
+    email: ${certmanager_email_address}
+    # Name of a secret used to store the ACME account private key
+    privateKeySecretRef:
+      name: letsencrypt-prod
+    # Enable the HTTP-01 challenge provider
+    solvers:
+    - http01:
+        ingress:
+          class: nginx
+EOF
+    kubectl create -f prod_issuer.yaml
+    kubectl create -f staging_issuer.yaml
+fi
+%{ endif }
+
 %{ endif }
 
 #kubectl get pods -n kube-system | grep aws-node | wc -l

diff --git a/k3s_cluster/vars.tf b/k3s_cluster/vars.tf
@@ -37,6 +37,26 @@ variable "my_public_ip_cidr" {
   description = "My public ip CIDR"
 }
 
+variable "install_nginx_ingress" {
+  type    = bool
+  default = true
+}
+
+variable "install_certmanager" {
+  type    = bool
+  default = true
+}
+
+variable "certmanager_release" {
+  type    = string
+  default = "v1.8.2"
+}
+
+variable "certmanager_email_address" {
+  type    = string
+  default = "changeme@example.com"
+}
+
 variable "vpc_subnet_cidr" {
   type        = string
   description = "VPC subnet CIDR"

diff --git a/main.tf b/main.tf
@@ -13,15 +13,17 @@ variable "AWS_REGION" {
 }
 
 module "k3s_cluster" {
-  AWS_REGION          = "<change_me>"
-  environment         = "staging"
-  my_public_ip_cidr   = "<change_me>"
-  vpc_id              = "<change_me>"
-  vpc_subnets         = ["<change_me>", "<change_me>", "<change_me>"]
-  vpc_subnet_cidr     = "<change_me>"
-  cluster_name        = "k3s-cluster"
-  k3s_token           = "<change_me>"
-  source              = "./k3s_cluster/"
+  AWS_REGION                = "<change_me>"
+  environment               = "staging"
+  my_public_ip_cidr         = "<change_me>"
+  vpc_id                    = "<change_me>"
+  vpc_subnets               = ["<change_me>", "<change_me>", "<change_me>"]
+  vpc_subnet_cidr           = "<change_me>"
+  cluster_name              = "k3s-cluster"
+  k3s_token                 = "<change_me>"
+  create_extlb              = true
+  certmanager_email_address = "<change_me>"
+  source                    = "./k3s_cluster/"
 }
 
 output "elb_dns_name" {