Change kube-apiserver port from 443 to 6443

* Adjust firewall rules, security groups, cloud load balancers, and generated kubeconfig's * Drop root privileges in apiserver pods * Facilitates some future simplifications and cost reductions * Bare-Metal users who exposed kube-apiserver on a WAN via their router or load balancer will need to adjust its configuration. This is uncommon, most apiserver are on LAN and/or behind VPN so no routing infrastructure is configured with the port number
poseidon · Jun 20, 2018 · db921e7 · db921e7
1 parent 6e64634
commit db921e7
Show file tree

Hide file tree

Showing 19 changed files with 43 additions and 27 deletions.
diff --git a/CHANGES.md b/CHANGES.md
@@ -4,10 +4,16 @@ Notable changes between versions.
 
 ## Latest
 
+* Change `kube-apiserver` port from 443 to 6443 ([#248](https://github.com/poseidon/typhoon/pull/248))
+  * Drop root privileges from `kube-apiserver` pods
+  * Adjust firewall rules, security groups, and generated kubeconfig's
+  * Facilitates some simplifications and cost reductions
 * Update etcd from v3.3.6 to v3.3.8 ([#243](https://github.com/poseidon/typhoon/pull/243), [#247](https://github.com/poseidon/typhoon/pull/247))
 
 #### Bare-Metal
 
+* Change `kube-apiserver` port from 443 to 6443 ([#248](https://github.com/poseidon/typhoon/pull/248))
+  * Adjust router or load balancer configuration for 6443 if you exposed apiserver on a WAN (uncommon, most apiservers are internal or behind VPN)
 * Fix possible deadlock when provisioning clusters larger than 10 nodes ([#244](https://github.com/poseidon/typhoon/pull/244)) 
 
 #### Addons

diff --git a/aws/container-linux/kubernetes/apiserver.tf b/aws/container-linux/kubernetes/apiserver.tf
@@ -28,7 +28,7 @@ resource "aws_lb" "apiserver" {
 resource "aws_lb_listener" "apiserver-https" {
   load_balancer_arn = "${aws_lb.apiserver.arn}"
   protocol          = "TCP"
-  port              = "443"
+  port              = "6443"
 
   default_action {
     type             = "forward"
@@ -43,12 +43,12 @@ resource "aws_lb_target_group" "controllers" {
   target_type = "instance"
 
   protocol = "TCP"
-  port     = 443
+  port     = 6443
 
   # TCP health check for apiserver
   health_check {
     protocol = "TCP"
-    port     = 443
+    port     = 6443
 
     # NLBs required to use same healthy and unhealthy thresholds
     healthy_threshold   = 3
@@ -65,5 +65,5 @@ resource "aws_lb_target_group_attachment" "controllers" {
 
   target_group_arn = "${aws_lb_target_group.controllers.arn}"
   target_id        = "${element(aws_instance.controllers.*.id, count.index)}"
-  port             = 443
+  port             = 6443
 }
diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf
@@ -1,6 +1,6 @@
 # Self-hosted Kubernetes assets (kubeconfig, manifests)
 module "bootkube" {
-  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1"
+  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=b455fb26678e401a3c1db87637b09cb5fba0e80d"
 
   cluster_name          = "${var.cluster_name}"
   api_servers           = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"]

diff --git a/aws/container-linux/kubernetes/security.tf b/aws/container-linux/kubernetes/security.tf
@@ -36,8 +36,8 @@ resource "aws_security_group_rule" "controller-apiserver" {
 
   type        = "ingress"
   protocol    = "tcp"
-  from_port   = 443
-  to_port     = 443
+  from_port   = 6443
+  to_port     = 6443
   cidr_blocks = ["0.0.0.0/0"]
 }
 

diff --git a/aws/fedora-atomic/kubernetes/bootkube.tf b/aws/fedora-atomic/kubernetes/bootkube.tf
@@ -1,6 +1,6 @@
 # Self-hosted Kubernetes assets (kubeconfig, manifests)
 module "bootkube" {
-  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1"
+  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=b455fb26678e401a3c1db87637b09cb5fba0e80d"
 
   cluster_name          = "${var.cluster_name}"
   api_servers           = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"]

diff --git a/aws/fedora-atomic/kubernetes/security.tf b/aws/fedora-atomic/kubernetes/security.tf
@@ -36,8 +36,8 @@ resource "aws_security_group_rule" "controller-apiserver" {
 
   type        = "ingress"
   protocol    = "tcp"
-  from_port   = 443
-  to_port     = 443
+  from_port   = 6443
+  to_port     = 6443
   cidr_blocks = ["0.0.0.0/0"]
 }
 

diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf
@@ -1,6 +1,6 @@
 # Self-hosted Kubernetes assets (kubeconfig, manifests)
 module "bootkube" {
-  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1"
+  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=b455fb26678e401a3c1db87637b09cb5fba0e80d"
 
   cluster_name                    = "${var.cluster_name}"
   api_servers                     = ["${var.k8s_domain_name}"]

diff --git a/bare-metal/fedora-atomic/kubernetes/bootkube.tf b/bare-metal/fedora-atomic/kubernetes/bootkube.tf
@@ -1,6 +1,6 @@
 # Self-hosted Kubernetes assets (kubeconfig, manifests)
 module "bootkube" {
-  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1"
+  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=b455fb26678e401a3c1db87637b09cb5fba0e80d"
 
   cluster_name          = "${var.cluster_name}"
   api_servers           = ["${var.k8s_domain_name}"]

diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf
@@ -1,6 +1,6 @@
 # Self-hosted Kubernetes assets (kubeconfig, manifests)
 module "bootkube" {
-  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1"
+  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=b455fb26678e401a3c1db87637b09cb5fba0e80d"
 
   cluster_name          = "${var.cluster_name}"
   api_servers           = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"]

diff --git a/digital-ocean/container-linux/kubernetes/network.tf b/digital-ocean/container-linux/kubernetes/network.tf
@@ -3,7 +3,7 @@ resource "digitalocean_firewall" "rules" {
 
   tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]
 
-  # allow ssh, http/https ingress, and peer-to-peer traffic
+  # allow ssh, apiserver, http/https ingress, and peer-to-peer traffic
   inbound_rule = [
     {
       protocol         = "tcp"
@@ -20,6 +20,11 @@ resource "digitalocean_firewall" "rules" {
       port_range       = "443"
       source_addresses = ["0.0.0.0/0", "::/0"]
     },
+    {
+      protocol         = "tcp"
+      port_range       = "6443"
+      source_addresses = ["0.0.0.0/0", "::/0"]
+    },
     {
       protocol    = "udp"
       port_range  = "1-65535"

diff --git a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf
@@ -1,6 +1,6 @@
 # Self-hosted Kubernetes assets (kubeconfig, manifests)
 module "bootkube" {
-  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1"
+  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=b455fb26678e401a3c1db87637b09cb5fba0e80d"
 
   cluster_name          = "${var.cluster_name}"
   api_servers           = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"]

diff --git a/digital-ocean/fedora-atomic/kubernetes/network.tf b/digital-ocean/fedora-atomic/kubernetes/network.tf
@@ -3,7 +3,7 @@ resource "digitalocean_firewall" "rules" {
 
   tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]
 
-  # allow ssh, http/https ingress, and peer-to-peer traffic
+  # allow ssh, apiserver, http/https ingress, and peer-to-peer traffic
   inbound_rule = [
     {
       protocol         = "tcp"
@@ -20,6 +20,11 @@ resource "digitalocean_firewall" "rules" {
       port_range       = "443"
       source_addresses = ["0.0.0.0/0", "::/0"]
     },
+    {
+      protocol         = "tcp"
+      port_range       = "6443"
+      source_addresses = ["0.0.0.0/0", "::/0"]
+    },
     {
       protocol    = "udp"
       port_range  = "1-65535"

diff --git a/docs/addons/ingress.md b/docs/addons/ingress.md
@@ -4,7 +4,7 @@ Nginx Ingress controller pods accept and demultiplex HTTP, HTTPS, TCP, or UDP tr
 
 ## AWS
 
-On AWS, an elastic load balancer distributes traffic across worker nodes (i.e. an auto-scaling group) running an Ingress controller deployment on host ports 80 and 443. Firewall rules allow traffic to ports 80 and 443. Health check rules ensure only workers with a health Ingress controller receive traffic.
+On AWS, a network load balancer (NLB) distributes traffic across a target group  of worker nodes running an Ingress controller deployment on host ports 80 and 443. Firewall rules allow traffic to ports 80 and 443. Health check rules ensure only workers with a health Ingress controller receive traffic.
 
 Create the Ingress controller deployment, service, RBAC roles, RBAC bindings, default backend, and namespace.
 

diff --git a/google-cloud/container-linux/kubernetes/apiserver.tf b/google-cloud/container-linux/kubernetes/apiserver.tf
@@ -23,7 +23,7 @@ resource "google_compute_global_forwarding_rule" "apiserver" {
   name        = "${var.cluster_name}-apiserver"
   ip_address  = "${google_compute_global_address.apiserver-ipv4.address}"
   ip_protocol = "TCP"
-  port_range  = "443"
+  port_range  = "6443"
   target      = "${google_compute_target_tcp_proxy.apiserver.self_link}"
 }
 
@@ -69,7 +69,7 @@ resource "google_compute_instance_group" "controllers" {
 
   named_port {
     name = "apiserver"
-    port = "443"
+    port = "6443"
   }
 
   # add instances in the zone into the instance group
@@ -92,6 +92,6 @@ resource "google_compute_health_check" "apiserver" {
   unhealthy_threshold = 3
 
   tcp_health_check {
-    port = "443"
+    port = "6443"
   }
 }
diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf
@@ -1,6 +1,6 @@
 # Self-hosted Kubernetes assets (kubeconfig, manifests)
 module "bootkube" {
-  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1"
+  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=b455fb26678e401a3c1db87637b09cb5fba0e80d"
 
   cluster_name          = "${var.cluster_name}"
   api_servers           = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"]

diff --git a/google-cloud/container-linux/kubernetes/network.tf b/google-cloud/container-linux/kubernetes/network.tf
@@ -23,7 +23,7 @@ resource "google_compute_firewall" "allow-apiserver" {
 
   allow {
     protocol = "tcp"
-    ports    = [443]
+    ports    = [6443]
   }
 
   source_ranges = ["0.0.0.0/0"]

diff --git a/google-cloud/fedora-atomic/kubernetes/apiserver.tf b/google-cloud/fedora-atomic/kubernetes/apiserver.tf
@@ -23,7 +23,7 @@ resource "google_compute_global_forwarding_rule" "apiserver" {
   name        = "${var.cluster_name}-apiserver"
   ip_address  = "${google_compute_global_address.apiserver-ipv4.address}"
   ip_protocol = "TCP"
-  port_range  = "443"
+  port_range  = "6443"
   target      = "${google_compute_target_tcp_proxy.apiserver.self_link}"
 }
 
@@ -69,7 +69,7 @@ resource "google_compute_instance_group" "controllers" {
 
   named_port {
     name = "apiserver"
-    port = "443"
+    port = "6443"
   }
 
   # add instances in the zone into the instance group
@@ -92,6 +92,6 @@ resource "google_compute_health_check" "apiserver" {
   unhealthy_threshold = 3
 
   tcp_health_check {
-    port = "443"
+    port = "6443"
   }
 }
diff --git a/google-cloud/fedora-atomic/kubernetes/bootkube.tf b/google-cloud/fedora-atomic/kubernetes/bootkube.tf
@@ -1,6 +1,6 @@
 # Self-hosted Kubernetes assets (kubeconfig, manifests)
 module "bootkube" {
-  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1"
+  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=b455fb26678e401a3c1db87637b09cb5fba0e80d"
 
   cluster_name          = "${var.cluster_name}"
   api_servers           = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"]

diff --git a/google-cloud/fedora-atomic/kubernetes/network.tf b/google-cloud/fedora-atomic/kubernetes/network.tf
@@ -23,7 +23,7 @@ resource "google_compute_firewall" "allow-apiserver" {
 
   allow {
     protocol = "tcp"
-    ports    = [443]
+    ports    = [6443]
   }
 
   source_ranges = ["0.0.0.0/0"]
-Original file line number
+Diff line change
@@ Expand Up @@
     ## AWS
-    On AWS, an elastic load balancer distributes traffic across worker nodes (i.e. an auto-scaling group) running an Ingress controller deployment on host ports 80 and 443. Firewall rules allow traffic to ports 80 and 443. Health check rules ensure only workers with a health Ingress controller receive traffic.
+    On AWS, a network load balancer (NLB) distributes traffic across a target group  of worker nodes running an Ingress controller deployment on host ports 80 and 443. Firewall rules allow traffic to ports 80 and 443. Health check rules ensure only workers with a health Ingress controller receive traffic.
     Create the Ingress controller deployment, service, RBAC roles, RBAC bindings, default backend, and namespace.
@@ Expand Down @@