From db921e75ff7058d3da1271876a984b24054eaf8e Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 18 Jun 2018 21:57:58 -0700 Subject: [PATCH] Change kube-apiserver port from 443 to 6443 * Adjust firewall rules, security groups, cloud load balancers, and generated kubeconfig's * Drop root privileges in apiserver pods * Facilitates some future simplifications and cost reductions * Bare-Metal users who exposed kube-apiserver on a WAN via their router or load balancer will need to adjust its configuration. This is uncommon, most apiserver are on LAN and/or behind VPN so no routing infrastructure is configured with the port number --- CHANGES.md | 6 ++++++ aws/container-linux/kubernetes/apiserver.tf | 8 ++++---- aws/container-linux/kubernetes/bootkube.tf | 2 +- aws/container-linux/kubernetes/security.tf | 4 ++-- aws/fedora-atomic/kubernetes/bootkube.tf | 2 +- aws/fedora-atomic/kubernetes/security.tf | 4 ++-- bare-metal/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/fedora-atomic/kubernetes/bootkube.tf | 2 +- digital-ocean/container-linux/kubernetes/bootkube.tf | 2 +- digital-ocean/container-linux/kubernetes/network.tf | 7 ++++++- digital-ocean/fedora-atomic/kubernetes/bootkube.tf | 2 +- digital-ocean/fedora-atomic/kubernetes/network.tf | 7 ++++++- docs/addons/ingress.md | 2 +- google-cloud/container-linux/kubernetes/apiserver.tf | 6 +++--- google-cloud/container-linux/kubernetes/bootkube.tf | 2 +- google-cloud/container-linux/kubernetes/network.tf | 2 +- google-cloud/fedora-atomic/kubernetes/apiserver.tf | 6 +++--- google-cloud/fedora-atomic/kubernetes/bootkube.tf | 2 +- google-cloud/fedora-atomic/kubernetes/network.tf | 2 +- 19 files changed, 43 insertions(+), 27 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 417c967a7..906f4adf4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,10 +4,16 @@ Notable changes between versions. ## Latest +* Change `kube-apiserver` port from 443 to 6443 ([#248](https://github.com/poseidon/typhoon/pull/248)) + * Drop root privileges from `kube-apiserver` pods + * Adjust firewall rules, security groups, and generated kubeconfig's + * Facilitates some simplifications and cost reductions * Update etcd from v3.3.6 to v3.3.8 ([#243](https://github.com/poseidon/typhoon/pull/243), [#247](https://github.com/poseidon/typhoon/pull/247)) #### Bare-Metal +* Change `kube-apiserver` port from 443 to 6443 ([#248](https://github.com/poseidon/typhoon/pull/248)) + * Adjust router or load balancer configuration for 6443 if you exposed apiserver on a WAN (uncommon, most apiservers are internal or behind VPN) * Fix possible deadlock when provisioning clusters larger than 10 nodes ([#244](https://github.com/poseidon/typhoon/pull/244)) #### Addons diff --git a/aws/container-linux/kubernetes/apiserver.tf b/aws/container-linux/kubernetes/apiserver.tf index 8cc5eed6e..93cbcee45 100644 --- a/aws/container-linux/kubernetes/apiserver.tf +++ b/aws/container-linux/kubernetes/apiserver.tf @@ -28,7 +28,7 @@ resource "aws_lb" "apiserver" { resource "aws_lb_listener" "apiserver-https" { load_balancer_arn = "${aws_lb.apiserver.arn}" protocol = "TCP" - port = "443" + port = "6443" default_action { type = "forward" @@ -43,12 +43,12 @@ resource "aws_lb_target_group" "controllers" { target_type = "instance" protocol = "TCP" - port = 443 + port = 6443 # TCP health check for apiserver health_check { protocol = "TCP" - port = 443 + port = 6443 # NLBs required to use same healthy and unhealthy thresholds healthy_threshold = 3 @@ -65,5 +65,5 @@ resource "aws_lb_target_group_attachment" "controllers" { target_group_arn = "${aws_lb_target_group.controllers.arn}" target_id = "${element(aws_instance.controllers.*.id, count.index)}" - port = 443 + port = 6443 } diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index 4da810ccc..5ac959425 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=b455fb26678e401a3c1db87637b09cb5fba0e80d" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/aws/container-linux/kubernetes/security.tf b/aws/container-linux/kubernetes/security.tf index 8534200be..8104080e6 100644 --- a/aws/container-linux/kubernetes/security.tf +++ b/aws/container-linux/kubernetes/security.tf @@ -36,8 +36,8 @@ resource "aws_security_group_rule" "controller-apiserver" { type = "ingress" protocol = "tcp" - from_port = 443 - to_port = 443 + from_port = 6443 + to_port = 6443 cidr_blocks = ["0.0.0.0/0"] } diff --git a/aws/fedora-atomic/kubernetes/bootkube.tf b/aws/fedora-atomic/kubernetes/bootkube.tf index 7f990934e..218c948f3 100644 --- a/aws/fedora-atomic/kubernetes/bootkube.tf +++ b/aws/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=b455fb26678e401a3c1db87637b09cb5fba0e80d" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/aws/fedora-atomic/kubernetes/security.tf b/aws/fedora-atomic/kubernetes/security.tf index 8534200be..8104080e6 100644 --- a/aws/fedora-atomic/kubernetes/security.tf +++ b/aws/fedora-atomic/kubernetes/security.tf @@ -36,8 +36,8 @@ resource "aws_security_group_rule" "controller-apiserver" { type = "ingress" protocol = "tcp" - from_port = 443 - to_port = 443 + from_port = 6443 + to_port = 6443 cidr_blocks = ["0.0.0.0/0"] } diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index 7662cb65c..939516fd1 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=b455fb26678e401a3c1db87637b09cb5fba0e80d" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/bare-metal/fedora-atomic/kubernetes/bootkube.tf b/bare-metal/fedora-atomic/kubernetes/bootkube.tf index df40c89f2..64d3889ae 100644 --- a/bare-metal/fedora-atomic/kubernetes/bootkube.tf +++ b/bare-metal/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=b455fb26678e401a3c1db87637b09cb5fba0e80d" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index 1195cae05..afa05271b 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=b455fb26678e401a3c1db87637b09cb5fba0e80d" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/digital-ocean/container-linux/kubernetes/network.tf b/digital-ocean/container-linux/kubernetes/network.tf index fc257a030..312d7966e 100644 --- a/digital-ocean/container-linux/kubernetes/network.tf +++ b/digital-ocean/container-linux/kubernetes/network.tf @@ -3,7 +3,7 @@ resource "digitalocean_firewall" "rules" { tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] - # allow ssh, http/https ingress, and peer-to-peer traffic + # allow ssh, apiserver, http/https ingress, and peer-to-peer traffic inbound_rule = [ { protocol = "tcp" @@ -20,6 +20,11 @@ resource "digitalocean_firewall" "rules" { port_range = "443" source_addresses = ["0.0.0.0/0", "::/0"] }, + { + protocol = "tcp" + port_range = "6443" + source_addresses = ["0.0.0.0/0", "::/0"] + }, { protocol = "udp" port_range = "1-65535" diff --git a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf index b9b57058b..d13b3f02f 100644 --- a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf +++ b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=b455fb26678e401a3c1db87637b09cb5fba0e80d" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/digital-ocean/fedora-atomic/kubernetes/network.tf b/digital-ocean/fedora-atomic/kubernetes/network.tf index fc257a030..312d7966e 100644 --- a/digital-ocean/fedora-atomic/kubernetes/network.tf +++ b/digital-ocean/fedora-atomic/kubernetes/network.tf @@ -3,7 +3,7 @@ resource "digitalocean_firewall" "rules" { tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] - # allow ssh, http/https ingress, and peer-to-peer traffic + # allow ssh, apiserver, http/https ingress, and peer-to-peer traffic inbound_rule = [ { protocol = "tcp" @@ -20,6 +20,11 @@ resource "digitalocean_firewall" "rules" { port_range = "443" source_addresses = ["0.0.0.0/0", "::/0"] }, + { + protocol = "tcp" + port_range = "6443" + source_addresses = ["0.0.0.0/0", "::/0"] + }, { protocol = "udp" port_range = "1-65535" diff --git a/docs/addons/ingress.md b/docs/addons/ingress.md index 754bec808..e6c9e81e0 100644 --- a/docs/addons/ingress.md +++ b/docs/addons/ingress.md @@ -4,7 +4,7 @@ Nginx Ingress controller pods accept and demultiplex HTTP, HTTPS, TCP, or UDP tr ## AWS -On AWS, an elastic load balancer distributes traffic across worker nodes (i.e. an auto-scaling group) running an Ingress controller deployment on host ports 80 and 443. Firewall rules allow traffic to ports 80 and 443. Health check rules ensure only workers with a health Ingress controller receive traffic. +On AWS, a network load balancer (NLB) distributes traffic across a target group of worker nodes running an Ingress controller deployment on host ports 80 and 443. Firewall rules allow traffic to ports 80 and 443. Health check rules ensure only workers with a health Ingress controller receive traffic. Create the Ingress controller deployment, service, RBAC roles, RBAC bindings, default backend, and namespace. diff --git a/google-cloud/container-linux/kubernetes/apiserver.tf b/google-cloud/container-linux/kubernetes/apiserver.tf index ffeb8cd63..10145357e 100644 --- a/google-cloud/container-linux/kubernetes/apiserver.tf +++ b/google-cloud/container-linux/kubernetes/apiserver.tf @@ -23,7 +23,7 @@ resource "google_compute_global_forwarding_rule" "apiserver" { name = "${var.cluster_name}-apiserver" ip_address = "${google_compute_global_address.apiserver-ipv4.address}" ip_protocol = "TCP" - port_range = "443" + port_range = "6443" target = "${google_compute_target_tcp_proxy.apiserver.self_link}" } @@ -69,7 +69,7 @@ resource "google_compute_instance_group" "controllers" { named_port { name = "apiserver" - port = "443" + port = "6443" } # add instances in the zone into the instance group @@ -92,6 +92,6 @@ resource "google_compute_health_check" "apiserver" { unhealthy_threshold = 3 tcp_health_check { - port = "443" + port = "6443" } } diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index 51ab437a8..7f2620fb8 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=b455fb26678e401a3c1db87637b09cb5fba0e80d" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/container-linux/kubernetes/network.tf b/google-cloud/container-linux/kubernetes/network.tf index 85412bf12..eb2513292 100644 --- a/google-cloud/container-linux/kubernetes/network.tf +++ b/google-cloud/container-linux/kubernetes/network.tf @@ -23,7 +23,7 @@ resource "google_compute_firewall" "allow-apiserver" { allow { protocol = "tcp" - ports = [443] + ports = [6443] } source_ranges = ["0.0.0.0/0"] diff --git a/google-cloud/fedora-atomic/kubernetes/apiserver.tf b/google-cloud/fedora-atomic/kubernetes/apiserver.tf index 002c553ef..61f48cddb 100644 --- a/google-cloud/fedora-atomic/kubernetes/apiserver.tf +++ b/google-cloud/fedora-atomic/kubernetes/apiserver.tf @@ -23,7 +23,7 @@ resource "google_compute_global_forwarding_rule" "apiserver" { name = "${var.cluster_name}-apiserver" ip_address = "${google_compute_global_address.apiserver-ipv4.address}" ip_protocol = "TCP" - port_range = "443" + port_range = "6443" target = "${google_compute_target_tcp_proxy.apiserver.self_link}" } @@ -69,7 +69,7 @@ resource "google_compute_instance_group" "controllers" { named_port { name = "apiserver" - port = "443" + port = "6443" } # add instances in the zone into the instance group @@ -92,6 +92,6 @@ resource "google_compute_health_check" "apiserver" { unhealthy_threshold = 3 tcp_health_check { - port = "443" + port = "6443" } } diff --git a/google-cloud/fedora-atomic/kubernetes/bootkube.tf b/google-cloud/fedora-atomic/kubernetes/bootkube.tf index 4a93e35df..b71a61346 100644 --- a/google-cloud/fedora-atomic/kubernetes/bootkube.tf +++ b/google-cloud/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=b455fb26678e401a3c1db87637b09cb5fba0e80d" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/fedora-atomic/kubernetes/network.tf b/google-cloud/fedora-atomic/kubernetes/network.tf index 85412bf12..eb2513292 100644 --- a/google-cloud/fedora-atomic/kubernetes/network.tf +++ b/google-cloud/fedora-atomic/kubernetes/network.tf @@ -23,7 +23,7 @@ resource "google_compute_firewall" "allow-apiserver" { allow { protocol = "tcp" - ports = [443] + ports = [6443] } source_ranges = ["0.0.0.0/0"]