From fb8c7abcecd4cdda9483e9e098072e1ff4fc8f74 Mon Sep 17 00:00:00 2001 From: smalltown Date: Mon, 20 Aug 2018 22:25:42 +0800 Subject: [PATCH 1/3] add necessary ignition, remove optional ignition --- aws/elastikube/ign-essential.tf | 4 +-- aws/elastikube/ign-optional.tf | 44 --------------------------------- aws/kube-etcd/ignition.tf | 17 +++++++++++-- aws/kube-master/ignition.tf | 13 ++++++++++ 4 files changed, 30 insertions(+), 48 deletions(-) delete mode 100644 aws/elastikube/ign-optional.tf diff --git a/aws/elastikube/ign-essential.tf b/aws/elastikube/ign-essential.tf index 4122e923..70d8ba5f 100644 --- a/aws/elastikube/ign-essential.tf +++ b/aws/elastikube/ign-essential.tf @@ -45,7 +45,7 @@ module "ignition_kube_addon_proxy" { resource "aws_security_group_rule" "master_ingress_flannel" { type = "ingress" - security_group_id = "${aws_security_group.master.id}" + security_group_id = "${module.master.master_sg_id}" protocol = "udp" from_port = 4789 @@ -55,7 +55,7 @@ resource "aws_security_group_rule" "master_ingress_flannel" { resource "aws_security_group_rule" "master_ingress_flannel_from_worker" { type = "ingress" - security_group_id = "${aws_security_group.master.id}" + security_group_id = "${module.master.master_sg_id}" source_security_group_id = "${aws_security_group.workers.id}" protocol = "udp" diff --git a/aws/elastikube/ign-optional.tf b/aws/elastikube/ign-optional.tf deleted file mode 100644 index 60d3e5dd..00000000 --- a/aws/elastikube/ign-optional.tf +++ /dev/null @@ -1,44 +0,0 @@ -# Optional components do not affect the functionality of Kubernetes. - -# --------------------------------------------------------------------------------------------------------------------- -# Prometheus Node Exporter -# --------------------------------------------------------------------------------------------------------------------- - -locals { - node_exporter_port = 9100 -} - -resource "aws_security_group_rule" "master_ingress_node_exporter" { - type = "ingress" - security_group_id = "${aws_security_group.master.id}" - - protocol = "udp" - from_port = "${local.node_exporter_port}" - to_port = "${local.node_exporter_port}" - self = true -} - -resource "aws_security_group_rule" "master_ingress_node_exporter_from_worker" { - type = "ingress" - security_group_id = "${aws_security_group.master.id}" - source_security_group_id = "${aws_security_group.workers.id}" - - protocol = "udp" - from_port = "${local.node_exporter_port}" - to_port = "${local.node_exporter_port}" -} - -module "ignition_node_exporter" { - source = "../../ignitions/node-exporter" - - listen_port = "${local.node_exporter_port}" -} - -# --------------------------------------------------------------------------------------------------------------------- -# Locksmithd - Reboot strategy -# --------------------------------------------------------------------------------------------------------------------- - -module "ignition_locksmithd" { - source = "../../ignitions/locksmithd" - reboot_strategy = "off" -} diff --git a/aws/kube-etcd/ignition.tf b/aws/kube-etcd/ignition.tf index f5cf3bd1..1481f91b 100644 --- a/aws/kube-etcd/ignition.tf +++ b/aws/kube-etcd/ignition.tf @@ -2,6 +2,15 @@ module "ignition_docker" { source = "../../ignitions/docker" } +module "ignition_locksmithd" { + source = "../../ignitions/locksmithd" + reboot_strategy = "${var.reboot_strategy}" +} + +module "ignition_update_ca_certificates" { + source = "../../ignitions/update-ca-certificates" +} + module "ignition_etcd" { source = "../../ignitions/etcd" @@ -24,16 +33,20 @@ module "ignition_etcd" { data "ignition_config" "main" { files = ["${compact(concat( module.ignition_docker.files, + module.ignition_locksmithd.files, + module.ignition_update_ca_certificates.files, module.ignition_etcd.files, module.ignition_node_exporter.files, - var.extra_ignition_file_ids, + var.extra_ignition_file_ids ))}"] systemd = ["${compact(concat( module.ignition_docker.systemd_units, + module.ignition_locksmithd.systemd_units, + module.ignition_update_ca_certificates.systemd_units, module.ignition_etcd.systemd_units, module.ignition_node_exporter.systemd_units, - var.extra_ignition_systemd_unit_ids, + var.extra_ignition_systemd_unit_ids ))}"] } diff --git a/aws/kube-master/ignition.tf b/aws/kube-master/ignition.tf index e5aa3013..4bac13a4 100644 --- a/aws/kube-master/ignition.tf +++ b/aws/kube-master/ignition.tf @@ -6,6 +6,15 @@ module "ignition_docker" { source = "../../ignitions/docker" } +module "ignition_locksmithd" { + source = "../../ignitions/locksmithd" + reboot_strategy = "${var.reboot_strategy}" +} + +module "ignition_update_ca_certificates" { + source = "../../ignitions/update-ca-certificates" +} + module "ignition_kube_config" { source = "../../ignitions/kube-config" @@ -22,7 +31,9 @@ module "ignition_kube_config" { data "ignition_config" "main" { files = ["${compact(concat( module.ignition_docker.files, + module.ignition_locksmithd.files, module.ignition_kube_control_plane.files, + module.ignition_update_ca_certificates.files, module.ignition_kubelet.files, module.ignition_kube_config.files, var.extra_ignition_file_ids, @@ -30,7 +41,9 @@ data "ignition_config" "main" { systemd = ["${compact(concat( module.ignition_docker.systemd_units, + module.ignition_locksmithd.systemd_units, module.ignition_kube_control_plane.systemd_units, + module.ignition_update_ca_certificates.systemd_units, module.ignition_kubelet.systemd_units, module.ignition_kube_config.systemd_units, var.extra_ignition_systemd_unit_ids, From 0b8520d5bc23891c4fcb20cc029e3daf39dcbbf3 Mon Sep 17 00:00:00 2001 From: smalltown Date: Mon, 20 Aug 2018 22:26:30 +0800 Subject: [PATCH 2/3] modify mater seciruty group --- aws/elastikube/etcd.tf | 12 +++--------- aws/elastikube/master.tf | 11 +++-------- aws/elastikube/outputs.tf | 4 ++++ aws/elastikube/sg.tf | 9 --------- aws/elastikube/variables.tf | 2 +- aws/elastikube/worker-sg.tf | 4 ++-- aws/kube-etcd/ign-node-exporter.tf | 12 +----------- aws/kube-master/ign-control-plane.tf | 2 +- aws/kube-master/ign-kubelet.tf | 4 ++-- aws/kube-master/master.tf | 2 +- aws/kube-master/outputs.tf | 4 ++++ ignitions/kubelet/resources/services/kubelet.service | 1 + 12 files changed, 23 insertions(+), 44 deletions(-) delete mode 100644 aws/elastikube/sg.tf diff --git a/aws/elastikube/etcd.tf b/aws/elastikube/etcd.tf index cf2ed690..b0ec2588 100644 --- a/aws/elastikube/etcd.tf +++ b/aws/elastikube/etcd.tf @@ -8,20 +8,14 @@ module "etcd" { etcd_config = "${var.etcd_config}" subnet_ids = ["${var.subnet_ids}"] - master_security_group_id = "${aws_security_group.master.id}" + master_security_group_id = "${module.master.master_sg_id}" zone_id = "${aws_route53_zone.private.zone_id}" s3_bucket = "${aws_s3_bucket.ignition.id}" reboot_strategy = "${var.reboot_strategy}" - extra_ignition_file_ids = [ - "${module.ignition_locksmithd.files}", - "${var.extra_etcd_ignition_file_ids}", - ] + extra_ignition_file_ids = "${var.extra_etcd_ignition_file_ids}" - extra_ignition_systemd_unit_ids = [ - "${module.ignition_locksmithd.systemd_units}", - "${var.extra_etcd_ignition_systemd_unit_ids}", - ] + extra_ignition_systemd_unit_ids = "${var.extra_etcd_ignition_systemd_unit_ids}" extra_tags = "${var.extra_tags}" } diff --git a/aws/elastikube/master.tf b/aws/elastikube/master.tf index 0f2b8824..4c7599b4 100644 --- a/aws/elastikube/master.tf +++ b/aws/elastikube/master.tf @@ -9,8 +9,7 @@ module "master" { role_name = "${var.role_name}" security_group_ids = [ - "${aws_security_group.master.id}", - "${var.security_group_ids}", + "${var.security_group_ids}" ] lb_security_group_ids = ["${var.lb_security_group_ids}"] @@ -49,9 +48,7 @@ module "master" { "${module.ignition_kube_addon_dns.files}", "${module.ignition_kube_addon_proxy.files}", "${module.ignition_kube_addon_flannel_vxlan.files}", - "${module.ignition_node_exporter.files}", - "${module.ignition_locksmithd.files}", - "${var.extra_ignition_file_ids}", + "${var.extra_ignition_file_ids}" ] extra_ignition_systemd_unit_ids = [ @@ -59,9 +56,7 @@ module "master" { "${module.ignition_kube_addon_dns.systemd_units}", "${module.ignition_kube_addon_proxy.systemd_units}", "${module.ignition_kube_addon_flannel_vxlan.systemd_units}", - "${module.ignition_node_exporter.systemd_units}", - "${module.ignition_locksmithd.systemd_units}", - "${var.extra_ignition_systemd_unit_ids}", + "${var.extra_ignition_systemd_unit_ids}" ] extra_tags = "${var.extra_tags}" diff --git a/aws/elastikube/outputs.tf b/aws/elastikube/outputs.tf index 1e947c89..b453b751 100644 --- a/aws/elastikube/outputs.tf +++ b/aws/elastikube/outputs.tf @@ -22,6 +22,10 @@ output "s3_bucket" { value = "${aws_s3_bucket.ignition.id}" } +output "master_sg_ids" { + value = ["${module.master.master_sg_id}"] +} + output "worker_sg_ids" { value = ["${aws_security_group.workers.id}"] } diff --git a/aws/elastikube/sg.tf b/aws/elastikube/sg.tf deleted file mode 100644 index 3e45a231..00000000 --- a/aws/elastikube/sg.tf +++ /dev/null @@ -1,9 +0,0 @@ -resource "aws_security_group" "master" { - name_prefix = "${var.name}-master-" - vpc_id = "${local.vpc_id}" - - tags = "${merge(map( - "Name", "${var.name}-master", - "kubernetes.io/cluster/${var.name}", "owned", - ), var.extra_tags)}" -} diff --git a/aws/elastikube/variables.tf b/aws/elastikube/variables.tf index 5f8a6544..56187a8f 100644 --- a/aws/elastikube/variables.tf +++ b/aws/elastikube/variables.tf @@ -114,7 +114,7 @@ variable "hostzone" { variable "reboot_strategy" { type = "string" - default = "etcd-lock" + default = "off" description = "(Optional) CoreOS reboot strategies on updates, two option here: etcd-lock or off" } diff --git a/aws/elastikube/worker-sg.tf b/aws/elastikube/worker-sg.tf index 6d68b0b2..c3a4b39e 100644 --- a/aws/elastikube/worker-sg.tf +++ b/aws/elastikube/worker-sg.tf @@ -35,7 +35,7 @@ resource "aws_security_group_rule" "workers_ingress_cluster" { type = "ingress" description = "Allow workers Kubelets and pods to receive communication from the cluster control plane." security_group_id = "${aws_security_group.workers.id}" - source_security_group_id = "${aws_security_group.master.id}" + source_security_group_id = "${module.master.master_sg_id}" protocol = "tcp" from_port = 1025 @@ -68,7 +68,7 @@ resource "aws_security_group_rule" "worker_ingress_flannel_from_master" { description = "Allow access from other master flannel." type = "ingress" security_group_id = "${aws_security_group.workers.id}" - source_security_group_id = "${aws_security_group.master.id}" + source_security_group_id = "${module.master.master_sg_id}" protocol = "udp" from_port = 4789 diff --git a/aws/kube-etcd/ign-node-exporter.tf b/aws/kube-etcd/ign-node-exporter.tf index 96ef798a..9fcd79e6 100644 --- a/aws/kube-etcd/ign-node-exporter.tf +++ b/aws/kube-etcd/ign-node-exporter.tf @@ -2,21 +2,11 @@ locals { node_exporter_port = 9100 } -resource "aws_security_group_rule" "ingress_node_exporter" { - type = "ingress" - security_group_id = "${aws_security_group.etcd.id}" - - protocol = "udp" - from_port = "${local.node_exporter_port}" - to_port = "${local.node_exporter_port}" - self = true -} - resource "aws_security_group_rule" "ingress_node_exporter_from_worker" { type = "ingress" security_group_id = "${aws_security_group.etcd.id}" - protocol = "udp" + protocol = "tcp" cidr_blocks = ["${data.aws_vpc.etcd.cidr_block}"] from_port = "${local.node_exporter_port}" to_port = "${local.node_exporter_port}" diff --git a/aws/kube-master/ign-control-plane.tf b/aws/kube-master/ign-control-plane.tf index e97ac2be..a7d0c221 100644 --- a/aws/kube-master/ign-control-plane.tf +++ b/aws/kube-master/ign-control-plane.tf @@ -1,6 +1,6 @@ resource "aws_security_group_rule" "master_ingress" { type = "ingress" - security_group_id = "${aws_security_group.master.id}" + security_group_id = "${local.master_sg_id}" protocol = "tcp" cidr_blocks = ["${data.aws_vpc.master.cidr_block}"] diff --git a/aws/kube-master/ign-kubelet.tf b/aws/kube-master/ign-kubelet.tf index 9defff6a..c4b49572 100644 --- a/aws/kube-master/ign-kubelet.tf +++ b/aws/kube-master/ign-kubelet.tf @@ -1,6 +1,6 @@ resource "aws_security_group_rule" "master_ingress_kubelet_secure" { type = "ingress" - security_group_id = "${aws_security_group.master.id}" + security_group_id = "${local.master_sg_id}" protocol = "tcp" from_port = 10255 @@ -10,7 +10,7 @@ resource "aws_security_group_rule" "master_ingress_kubelet_secure" { resource "aws_security_group_rule" "master_ingress_kubelet_secure_from_worker" { type = "ingress" - security_group_id = "${aws_security_group.master.id}" + security_group_id = "${local.master_sg_id}" protocol = "tcp" cidr_blocks = ["${data.aws_vpc.master.cidr_block}"] diff --git a/aws/kube-master/master.tf b/aws/kube-master/master.tf index c03dcef1..5c022200 100644 --- a/aws/kube-master/master.tf +++ b/aws/kube-master/master.tf @@ -51,7 +51,7 @@ resource "aws_launch_configuration" "master" { security_groups = [ "${var.security_group_ids}", - "${aws_security_group.master.id}", + "${local.master_sg_id}", ] iam_instance_profile = "${aws_iam_instance_profile.master.id}" diff --git a/aws/kube-master/outputs.tf b/aws/kube-master/outputs.tf index 67f5381a..f0f6494c 100644 --- a/aws/kube-master/outputs.tf +++ b/aws/kube-master/outputs.tf @@ -5,3 +5,7 @@ output "certificate_authority" { output "endpoint" { value = "https://${aws_elb.master_internal.dns_name}" } + +output "master_sg_id" { + value = "${local.master_sg_id}" +} diff --git a/ignitions/kubelet/resources/services/kubelet.service b/ignitions/kubelet/resources/services/kubelet.service index 7a0f61fa..fdae465c 100644 --- a/ignitions/kubelet/resources/services/kubelet.service +++ b/ignitions/kubelet/resources/services/kubelet.service @@ -32,6 +32,7 @@ ExecStart=/usr/lib/coreos/kubelet-wrapper \ --minimum-container-ttl-duration=6m0s \ --client-ca-file=/etc/kubernetes/ca.crt \ --cluster-domain=cluster.local \ + --authentication-token-webhook=true \ ${kubelet_flag_cloud_provider} \ ${kubelet_flag_node_labels} \ ${kubelet_flag_register_with_taints} \ From f1dec854fd005a3821b44d1840c063d85836c97e Mon Sep 17 00:00:00 2001 From: smalltown Date: Tue, 21 Aug 2018 19:38:22 +0800 Subject: [PATCH 3/3] add kubelet_flag_extra_flags for each module --- aws/elastikube/master.tf | 2 ++ aws/elastikube/variables.tf | 6 ++++++ aws/kube-master/ign-kubelet.tf | 1 + aws/kube-master/variables.tf | 6 ++++++ aws/kube-worker-general/ignition.tf | 1 + aws/kube-worker-general/variables.tf | 6 ++++++ aws/kube-worker-spot/ignition.tf | 1 + aws/kube-worker-spot/variables.tf | 6 ++++++ ignitions/kubelet/resources/services/kubelet.service | 1 - 9 files changed, 29 insertions(+), 1 deletion(-) diff --git a/aws/elastikube/master.tf b/aws/elastikube/master.tf index 4c7599b4..c82b8c83 100644 --- a/aws/elastikube/master.tf +++ b/aws/elastikube/master.tf @@ -59,5 +59,7 @@ module "master" { "${var.extra_ignition_systemd_unit_ids}" ] + kubelet_flag_extra_flags = "${var.kubelet_flag_extra_flags}" + extra_tags = "${var.extra_tags}" } diff --git a/aws/elastikube/variables.tf b/aws/elastikube/variables.tf index 56187a8f..78a11bf2 100644 --- a/aws/elastikube/variables.tf +++ b/aws/elastikube/variables.tf @@ -157,6 +157,12 @@ variable "extra_ignition_systemd_unit_ids" { description = "(Optional) Additional ignition systemd unit IDs for masters. See https://www.terraform.io/docs/providers/ignition/d/systemd_unit.html for more details." } +variable "kubelet_flag_extra_flags" { + type = "list" + default = [] + description = "Extra user-provided flags to kubelet." +} + variable "extra_tags" { description = "(Optional) Extra AWS tags to be applied to the resources." type = "map" diff --git a/aws/kube-master/ign-kubelet.tf b/aws/kube-master/ign-kubelet.tf index c4b49572..bdf1eb1d 100644 --- a/aws/kube-master/ign-kubelet.tf +++ b/aws/kube-master/ign-kubelet.tf @@ -25,6 +25,7 @@ module "ignition_kubelet" { kubelet_flag_cluster_dns = "${local.cluster_dns_ip}" kubelet_flag_node_labels = "${join(",", var.kube_node_labels)}" kubelet_flag_register_with_taints = "${join(",", var.kube_node_taints)}" + kubelet_flag_extra_flags = "${var.kubelet_flag_extra_flags}" hyperkube = { image_path = "quay.io/coreos/hyperkube" diff --git a/aws/kube-master/variables.tf b/aws/kube-master/variables.tf index e15f281a..0ddcbf64 100644 --- a/aws/kube-master/variables.tf +++ b/aws/kube-master/variables.tf @@ -158,6 +158,12 @@ variable "extra_ignition_systemd_unit_ids" { description = "(Optional) Additional ignition systemd unit IDs. See https://www.terraform.io/docs/providers/ignition/d/systemd_unit.html for more details." } +variable "kubelet_flag_extra_flags" { + type = "list" + default = [] + description = "Extra user-provided flags to kubelet." +} + variable "extra_tags" { type = "map" default = {} diff --git a/aws/kube-worker-general/ignition.tf b/aws/kube-worker-general/ignition.tf index b94462e0..32760df0 100644 --- a/aws/kube-worker-general/ignition.tf +++ b/aws/kube-worker-general/ignition.tf @@ -37,6 +37,7 @@ module "ignition_kubelet" { )))}" kubelet_flag_register_with_taints = "${join(",", var.kube_node_taints)}" + kubelet_flag_extra_flags = "${var.kubelet_flag_extra_flags}" hyperkube = { image_path = "quay.io/coreos/hyperkube" diff --git a/aws/kube-worker-general/variables.tf b/aws/kube-worker-general/variables.tf index 9d3dc7a3..d7a1dcc5 100644 --- a/aws/kube-worker-general/variables.tf +++ b/aws/kube-worker-general/variables.tf @@ -123,6 +123,12 @@ variable "target_group_arns" { description = "(Optional) A list of aws_alb_target_group ARNs, for use with Application Load Balancing." } +variable "kubelet_flag_extra_flags" { + type = "list" + default = [] + description = "Extra user-provided flags to kubelet." +} + variable "extra_tags" { type = "map" default = {} diff --git a/aws/kube-worker-spot/ignition.tf b/aws/kube-worker-spot/ignition.tf index b94462e0..32760df0 100644 --- a/aws/kube-worker-spot/ignition.tf +++ b/aws/kube-worker-spot/ignition.tf @@ -37,6 +37,7 @@ module "ignition_kubelet" { )))}" kubelet_flag_register_with_taints = "${join(",", var.kube_node_taints)}" + kubelet_flag_extra_flags = "${var.kubelet_flag_extra_flags}" hyperkube = { image_path = "quay.io/coreos/hyperkube" diff --git a/aws/kube-worker-spot/variables.tf b/aws/kube-worker-spot/variables.tf index 41948d4a..e8cc0f32 100644 --- a/aws/kube-worker-spot/variables.tf +++ b/aws/kube-worker-spot/variables.tf @@ -125,6 +125,12 @@ variable "target_group_arns" { description = "(Optional) A list of aws_alb_target_group ARNs, for use with Application Load Balancing." } +variable "kubelet_flag_extra_flags" { + type = "list" + default = [] + description = "Extra user-provided flags to kubelet." +} + variable "extra_tags" { type = "map" default = {} diff --git a/ignitions/kubelet/resources/services/kubelet.service b/ignitions/kubelet/resources/services/kubelet.service index fdae465c..7a0f61fa 100644 --- a/ignitions/kubelet/resources/services/kubelet.service +++ b/ignitions/kubelet/resources/services/kubelet.service @@ -32,7 +32,6 @@ ExecStart=/usr/lib/coreos/kubelet-wrapper \ --minimum-container-ttl-duration=6m0s \ --client-ca-file=/etc/kubernetes/ca.crt \ --cluster-domain=cluster.local \ - --authentication-token-webhook=true \ ${kubelet_flag_cloud_provider} \ ${kubelet_flag_node_labels} \ ${kubelet_flag_register_with_taints} \