Add experimental Cilium CNI provider

* Accept "cilium" as an experimental CNI `networking` mode using overlay vxlan tunnels * Like most experiments, the feature is not listed in docs or supported, it may be removed at any time without notice * Platform firewalls: * Require UDP 8472 for vxlan (Linux kernel default) between nodes * Optional ICMP echo(8) between nodes for host reachability (health) * Optional TCP 4240 between nodes for host reachability (health) * Optional health ports are not opened on Azure * https://github.com/cilium/cilium * poseidon/terraform-render-bootstrap#192
poseidon · Jun 18, 2020 · e6c7463 · e6c7463
1 parent c25c590
commit e6c7463
Show file tree

Hide file tree

Showing 18 changed files with 502 additions and 52 deletions.
diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf
@@ -1,6 +1,6 @@
 # Kubernetes assets (kubeconfig, manifests)
 module "bootstrap" {
-  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3fe903d0accd71d198415cf46f2f6f53c5c4f699"
+  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=93210ba112091a1ef198ef1884749a46183a21a4"
 
   cluster_name          = var.cluster_name
   api_servers           = [format("%s.%s", var.cluster_name, var.dns_zone)]

diff --git a/aws/container-linux/kubernetes/security.tf b/aws/container-linux/kubernetes/security.tf
@@ -13,6 +13,30 @@ resource "aws_security_group" "controller" {
   }
 }
 
+resource "aws_security_group_rule" "controller-icmp" {
+  count = var.networking == "cilium" ? 1 : 0
+
+  security_group_id = aws_security_group.controller.id
+
+  type                     = "ingress"
+  protocol                 = "icmp"
+  from_port                = 8
+  to_port                  = 0
+  source_security_group_id = aws_security_group.worker.id
+}
+
+resource "aws_security_group_rule" "controller-icmp-self" {
+  count = var.networking == "cilium" ? 1 : 0
+
+  security_group_id = aws_security_group.controller.id
+
+  type      = "ingress"
+  protocol  = "icmp"
+  from_port = 8
+  to_port   = 0
+  self      = true
+}
+
 resource "aws_security_group_rule" "controller-ssh" {
   security_group_id = aws_security_group.controller.id
 
@@ -44,39 +68,31 @@ resource "aws_security_group_rule" "controller-etcd-metrics" {
   source_security_group_id = aws_security_group.worker.id
 }
 
-# Allow Prometheus to scrape kube-proxy
-resource "aws_security_group_rule" "kube-proxy-metrics" {
-  security_group_id = aws_security_group.controller.id
+resource "aws_security_group_rule" "controller-cilium-health" {
+  count = var.networking == "cilium" ? 1 : 0
 
-  type                     = "ingress"
-  protocol                 = "tcp"
-  from_port                = 10249
-  to_port                  = 10249
-  source_security_group_id = aws_security_group.worker.id
-}
-
-# Allow Prometheus to scrape kube-scheduler
-resource "aws_security_group_rule" "controller-scheduler-metrics" {
   security_group_id = aws_security_group.controller.id
 
   type                     = "ingress"
   protocol                 = "tcp"
-  from_port                = 10251
-  to_port                  = 10251
+  from_port                = 4240
+  to_port                  = 4240
   source_security_group_id = aws_security_group.worker.id
 }
 
-# Allow Prometheus to scrape kube-controller-manager
-resource "aws_security_group_rule" "controller-manager-metrics" {
+resource "aws_security_group_rule" "controller-cilium-health-self" {
+  count = var.networking == "cilium" ? 1 : 0
+
   security_group_id = aws_security_group.controller.id
 
-  type                     = "ingress"
-  protocol                 = "tcp"
-  from_port                = 10252
-  to_port                  = 10252
-  source_security_group_id = aws_security_group.worker.id
+  type      = "ingress"
+  protocol  = "tcp"
+  from_port = 4240
+  to_port   = 4240
+  self      = true
 }
 
+# IANA VXLAN default
 resource "aws_security_group_rule" "controller-vxlan" {
   count = var.networking == "flannel" ? 1 : 0
 
@@ -111,6 +127,31 @@ resource "aws_security_group_rule" "controller-apiserver" {
   cidr_blocks = ["0.0.0.0/0"]
 }
 
+# Linux VXLAN default
+resource "aws_security_group_rule" "controller-linux-vxlan" {
+  count = var.networking == "cilium" ? 1 : 0
+
+  security_group_id = aws_security_group.controller.id
+
+  type                     = "ingress"
+  protocol                 = "udp"
+  from_port                = 8472
+  to_port                  = 8472
+  source_security_group_id = aws_security_group.worker.id
+}
+
+resource "aws_security_group_rule" "controller-linux-vxlan-self" {
+  count = var.networking == "cilium" ? 1 : 0
+
+  security_group_id = aws_security_group.controller.id
+
+  type      = "ingress"
+  protocol  = "udp"
+  from_port = 8472
+  to_port   = 8472
+  self      = true
+}
+
 # Allow Prometheus to scrape node-exporter daemonset
 resource "aws_security_group_rule" "controller-node-exporter" {
   security_group_id = aws_security_group.controller.id
@@ -122,6 +163,17 @@ resource "aws_security_group_rule" "controller-node-exporter" {
   source_security_group_id = aws_security_group.worker.id
 }
 
+# Allow Prometheus to scrape kube-proxy
+resource "aws_security_group_rule" "kube-proxy-metrics" {
+  security_group_id = aws_security_group.controller.id
+
+  type                     = "ingress"
+  protocol                 = "tcp"
+  from_port                = 10249
+  to_port                  = 10249
+  source_security_group_id = aws_security_group.worker.id
+}
+
 # Allow apiserver to access kubelets for exec, log, port-forward
 resource "aws_security_group_rule" "controller-kubelet" {
   security_group_id = aws_security_group.controller.id
@@ -143,6 +195,28 @@ resource "aws_security_group_rule" "controller-kubelet-self" {
   self      = true
 }
 
+# Allow Prometheus to scrape kube-scheduler
+resource "aws_security_group_rule" "controller-scheduler-metrics" {
+  security_group_id = aws_security_group.controller.id
+
+  type                     = "ingress"
+  protocol                 = "tcp"
+  from_port                = 10251
+  to_port                  = 10251
+  source_security_group_id = aws_security_group.worker.id
+}
+
+# Allow Prometheus to scrape kube-controller-manager
+resource "aws_security_group_rule" "controller-manager-metrics" {
+  security_group_id = aws_security_group.controller.id
+
+  type                     = "ingress"
+  protocol                 = "tcp"
+  from_port                = 10252
+  to_port                  = 10252
+  source_security_group_id = aws_security_group.worker.id
+}
+
 resource "aws_security_group_rule" "controller-bgp" {
   security_group_id = aws_security_group.controller.id
 
@@ -227,6 +301,30 @@ resource "aws_security_group" "worker" {
   }
 }
 
+resource "aws_security_group_rule" "worker-icmp" {
+  count = var.networking == "cilium" ? 1 : 0
+
+  security_group_id = aws_security_group.worker.id
+
+  type                     = "ingress"
+  protocol                 = "icmp"
+  from_port                = 8
+  to_port                  = 0
+  source_security_group_id = aws_security_group.controller.id
+}
+
+resource "aws_security_group_rule" "worker-icmp-self" {
+  count = var.networking == "cilium" ? 1 : 0
+
+  security_group_id = aws_security_group.worker.id
+
+  type      = "ingress"
+  protocol  = "icmp"
+  from_port = 8
+  to_port   = 0
+  self      = true
+}
+
 resource "aws_security_group_rule" "worker-ssh" {
   security_group_id = aws_security_group.worker.id
 
@@ -257,6 +355,31 @@ resource "aws_security_group_rule" "worker-https" {
   cidr_blocks = ["0.0.0.0/0"]
 }
 
+resource "aws_security_group_rule" "worker-cilium-health" {
+  count = var.networking == "cilium" ? 1 : 0
+
+  security_group_id = aws_security_group.worker.id
+
+  type                     = "ingress"
+  protocol                 = "tcp"
+  from_port                = 4240
+  to_port                  = 4240
+  source_security_group_id = aws_security_group.controller.id
+}
+
+resource "aws_security_group_rule" "worker-cilium-health-self" {
+  count = var.networking == "cilium" ? 1 : 0
+
+  security_group_id = aws_security_group.worker.id
+
+  type      = "ingress"
+  protocol  = "tcp"
+  from_port = 4240
+  to_port   = 4240
+  self      = true
+}
+
+# IANA VXLAN default
 resource "aws_security_group_rule" "worker-vxlan" {
   count = var.networking == "flannel" ? 1 : 0
 
@@ -281,6 +404,31 @@ resource "aws_security_group_rule" "worker-vxlan-self" {
   self      = true
 }
 
+# Linux VXLAN default
+resource "aws_security_group_rule" "worker-linux-vxlan" {
+  count = var.networking == "cilium" ? 1 : 0
+
+  security_group_id = aws_security_group.worker.id
+
+  type                     = "ingress"
+  protocol                 = "udp"
+  from_port                = 8472
+  to_port                  = 8472
+  source_security_group_id = aws_security_group.controller.id
+}
+
+resource "aws_security_group_rule" "worker-linux-vxlan-self" {
+  count = var.networking == "cilium" ? 1 : 0
+
+  security_group_id = aws_security_group.worker.id
+
+  type      = "ingress"
+  protocol  = "udp"
+  from_port = 8472
+  to_port   = 8472
+  self      = true
+}
+
 # Allow Prometheus to scrape node-exporter daemonset
 resource "aws_security_group_rule" "worker-node-exporter" {
   security_group_id = aws_security_group.worker.id

diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf
@@ -1,6 +1,6 @@
 # Kubernetes assets (kubeconfig, manifests)
 module "bootstrap" {
-  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3fe903d0accd71d198415cf46f2f6f53c5c4f699"
+  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=93210ba112091a1ef198ef1884749a46183a21a4"
 
   cluster_name          = var.cluster_name
   api_servers           = [format("%s.%s", var.cluster_name, var.dns_zone)]