Merge pull request #9 from getamis/feature/spot_instance

Self-Hosted K8S Support Spot Worker Group

aws/elastikube/etcd.tf

@@ -11,6 +11,7 @@ module "etcd" {
   master_security_group_id = "${aws_security_group.master.id}"
   zone_id                  = "${aws_route53_zone.private.zone_id}"
   s3_bucket                = "${aws_s3_bucket.ignition.id}"
+  reboot_strategy          = "${var.reboot_strategy}"
 
   extra_ignition_file_ids         = ["${var.extra_etcd_ignition_file_ids}"]
   extra_ignition_systemd_unit_ids = ["${var.extra_etcd_ignition_systemd_unit_ids}"]

aws/elastikube/master.tf

@@ -36,6 +36,7 @@ module "master" {
   ]
 
   s3_bucket                       = "${aws_s3_bucket.ignition.id}"
+  reboot_strategy                 = "${var.reboot_strategy}"
   extra_ignition_file_ids         = ["${var.extra_ignition_file_ids}"]
   extra_ignition_systemd_unit_ids = ["${var.extra_ignition_systemd_unit_ids}"]
 

aws/elastikube/outputs.tf

@@ -25,3 +25,11 @@ output "s3_bucket" {
 output "worker_sg_ids" {
   value = ["${module.master.worker_sg_ids}"]
 }
+
+output "spot_fleet_tagging_role_arn" {
+  value = "${module.master.spot_fleet_tagging_role_arn}"
+}
+
+output "spot_fleet_autoscale_role_arn" {
+  value = "${module.master.spot_fleet_autoscale_role_arn}"
+}

aws/elastikube/variables.tf

@@ -102,6 +102,12 @@ variable "hostzone" {
   description = "(Optional) The cluster private hostname. If not specified, <cluster name>.com will be used."
 }
 
+variable "reboot_strategy" {
+  type    = "string"
+  default = "etcd-lock"
+  description = "(Optional) CoreOS reboot strategies on updates, two option here: etcd-lock or off"
+}
+
 variable "extra_master_node_labels" {
   type        = "list"
   default     = []

aws/ignitions/locksmithd/assets.tf

@@ -0,0 +1,8 @@
+locals {
+  mark = "${ var.reboot_strategy == "off" ? true : false }"
+}
+
+data "ignition_systemd_unit" "locksmithd" {
+  name = "locksmithd.service"
+  mask = "${local.mark}"
+}

aws/ignitions/locksmithd/outputs.tf

@@ -0,0 +1,9 @@
+output "systemd_units" {
+  value = [
+    "${data.ignition_systemd_unit.locksmithd.id}",
+  ]
+}
+
+output "files" {
+  value = []
+}

aws/ignitions/locksmithd/variables.tf

@@ -0,0 +1,4 @@
+variable "reboot_strategy" {
+  type    = "string"
+  description = "(Optional) CoreOS reboot strategies on updates, two option here: etcd-lock or off"
+}

aws/kube-etcd/ignition.tf

@@ -25,26 +25,29 @@ module "ignition_node_exporter" {
   source = "../ignitions/node-exporter"
 }
 
+module "ignition_locksmithd" {
+  source          = "../ignitions/locksmithd"
+  reboot_strategy = "${var.reboot_strategy}"
+}
+
 data "ignition_config" "main" {
   files = ["${compact(concat(
     module.ignition_docker.files,
     module.ignition_etcd.files,
     module.ignition_node_exporter.files,
+    module.ignition_locksmithd.files,
     var.extra_ignition_file_ids,
   ))}"]
 
   systemd = ["${compact(concat(
     module.ignition_docker.systemd_units,
     module.ignition_etcd.systemd_units,
     module.ignition_node_exporter.systemd_units,
+    module.ignition_locksmithd.systemd_units,
     var.extra_ignition_systemd_unit_ids,
   ))}"]
 }
 
-data "aws_region" "current" {
-  current = true
-}
-
 resource "aws_s3_bucket_object" "ignition" {
   bucket  = "${var.s3_bucket}"
   key     = "ign-etcd-${var.name}.json"

aws/kube-etcd/variables.tf

@@ -117,6 +117,11 @@ variable "s3_bucket" {
 EOF
 }
 
+variable "reboot_strategy" {
+  type    = "string"
+  description = "(Optional) CoreOS reboot strategies on updates, two option here: etcd-lock or off"
+}
+
 variable "extra_ignition_file_ids" {
   type        = "list"
   default     = []

aws/kube-master/ignition.tf

@@ -6,6 +6,11 @@ module "ignition_docker" {
   source = "../ignitions/docker"
 }
 
+module "ignition_locksmithd" {
+  source          = "../ignitions/locksmithd"
+  reboot_strategy = "${var.reboot_strategy}"
+}
+
 module "ignition_kube_config" {
   source = "../ignitions/kube-config"
 
@@ -23,6 +28,7 @@ data "ignition_config" "main" {
   files = ["${compact(concat(
     module.ignition_docker.files,
     module.ignition_node_exporter.files,
+    module.ignition_locksmithd.files,
     module.ignition_kube_control_plane.files,
     module.ignition_kubelet.files,
     module.ignition_kube_config.files,
@@ -36,6 +42,7 @@ data "ignition_config" "main" {
   systemd = ["${compact(concat(
     module.ignition_docker.systemd_units,
     module.ignition_node_exporter.systemd_units,
+    module.ignition_locksmithd.systemd_units,
     module.ignition_kube_control_plane.systemd_units,
     module.ignition_kubelet.systemd_units,
     module.ignition_kube_config.systemd_units,

aws/kube-master/outputs.tf

@@ -9,3 +9,11 @@ output "endpoint" {
 output "worker_sg_ids" {
   value = ["${aws_security_group.workers.id}"]
 }
+
+output "spot_fleet_tagging_role_arn" {
+  value = "${aws_iam_role.spot_fleet_tagging.arn}"
+}
+
+output "spot_fleet_autoscale_role_arn" {
+  value = "${aws_iam_role.spot_fleet_autoscale.arn}"
+}

aws/kube-master/role.tf

@@ -76,3 +76,55 @@ resource "aws_iam_role_policy_attachment" "master" {
   policy_arn = "${aws_iam_policy.master.arn}"
   role       = "${aws_iam_role.master.name}"
 }
+
+# Role for Spot Fleet
+resource "aws_iam_role" "spot_fleet_tagging" {
+  name = "${var.name}-spot-fleet-tagging"
+
+  assume_role_policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Action": "sts:AssumeRole",
+      "Principal": {
+        "Service": "spotfleet.amazonaws.com"
+      },
+      "Effect": "Allow",
+      "Sid": ""
+    }
+  ]
+}
+EOF
+}
+
+resource "aws_iam_role_policy_attachment" "spot_fleet_tagging" {
+  role       = "${aws_iam_role.spot_fleet_tagging.name}"
+  policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonEC2SpotFleetTaggingRole"
+}
+
+
+resource "aws_iam_role" "spot_fleet_autoscale" {
+  name = "${var.name}-spot-fleet-autoscale"
+
+  assume_role_policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Action": "sts:AssumeRole",
+      "Principal": {
+        "Service": "ec2.amazonaws.com"
+      },
+      "Effect": "Allow",
+      "Sid": ""
+    }
+  ]
+}
+EOF
+}
+
+resource "aws_iam_role_policy_attachment" "spot_fleet_autoscale" {
+  role       = "${aws_iam_role.spot_fleet_autoscale.name}"
+  policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonEC2SpotFleetAutoscaleRole"
+}

aws/kube-master/variables.tf

@@ -131,6 +131,11 @@ variable "s3_bucket" {
 EOF
 }
 
+variable "reboot_strategy" {
+  type    = "string"
+  description = "(Optional) CoreOS reboot strategies on updates, two option here: etcd-lock or off"
+}
+
 variable "extra_ignition_file_ids" {
   type        = "list"
   default     = []

aws/kube-worker-general/ignition.tf

@@ -6,6 +6,11 @@ module "ignition_docker" {
   source = "../ignitions/docker"
 }
 
+module "ignition_locksmithd" {
+  source          = "../ignitions/locksmithd"
+  reboot_strategy = "${var.reboot_strategy}"
+}
+
 data "aws_s3_bucket_object" "kubeconfig" {
   bucket = "${var.s3_bucket}"
   key    = "kubeconfig"
@@ -23,7 +28,7 @@ module "ignition_kubelet" {
   kubelet_flag_cluster_dns    = "${local.cluster_dns_ip}"
 
   kubelet_flag_node_labels = "${join(",", compact(concat(
-    list("node-role.kubernetes.io/general"),
+    list("node-role.kubernetes.io/${var.worker_config["name"]}"),
     var.kube_node_labels,
   )))}"
 
@@ -38,13 +43,15 @@ module "ignition_kubelet" {
 data "ignition_config" "main" {
   files = ["${compact(concat(
     module.ignition_docker.files,
+    module.ignition_locksmithd.files,
     module.ignition_kubelet.files,
     module.ignition_kube_config.files,
     var.extra_ignition_file_ids,
   ))}"]
 
   systemd = ["${compact(concat(
     module.ignition_docker.systemd_units,
+    module.ignition_locksmithd.systemd_units,
     module.ignition_kubelet.systemd_units,
     module.ignition_kube_config.systemd_units,
     var.extra_ignition_systemd_unit_ids,
@@ -53,14 +60,14 @@ data "ignition_config" "main" {
 
 resource "aws_s3_bucket_object" "ignition" {
   bucket  = "${var.s3_bucket}"
-  key     = "ign-worker-general-${var.name}.json"
+  key     = "ign-worker-${var.worker_config["name"]}-${var.name}.json"
   content = "${data.ignition_config.main.rendered}"
   acl     = "private"
 
   server_side_encryption = "AES256"
 
   tags = "${merge(map(
-      "Name", "ign-worker-general-${var.name}.json",
+      "Name", "ign-worker-${var.worker_config["name"]}-${var.name}.json",
       "Role", "worker",
       "kubernetes.io/cluster/${var.name}", "owned",
     ), var.extra_tags)}"

aws/kube-worker-general/role.tf

@@ -24,7 +24,7 @@ resource "aws_iam_role" "worker" {
 }
 
 resource "aws_iam_instance_profile" "worker" {
-  name = "${var.name}-worker"
+  name = "${var.name}-worker-${var.worker_config["name"]}"
 
   role = "${var.role_arn == "" ?
     join("|", aws_iam_role.worker.*.name) :
@@ -34,7 +34,7 @@ resource "aws_iam_instance_profile" "worker" {
 
 resource "aws_iam_policy" "worker" {
   count       = "${var.role_arn == "" ? 1 : 0}"
-  name        = "${var.name}-worker"
+  name        = "${var.name}-worker-${var.worker_config["name"]}"
   path        = "/"
   description = "policy for kubernetes workers"
 

aws/kube-worker-general/variables.tf

@@ -48,6 +48,7 @@ variable "worker_config" {
   default = {
     instance_count   = "1"
     ec2_type         = "t2.medium"
+    name             = "general"
     root_volume_iops = "100"
     root_volume_size = "256"
     root_volume_type = "gp2"
@@ -92,6 +93,12 @@ variable "s3_bucket" {
 EOF
 }
 
+variable "reboot_strategy" {
+  type    = "string"
+  default = "etcd-lock"
+  description = "(Optional) CoreOS reboot strategies on updates, two option here: etcd-lock or off"
+}
+
 variable "extra_ignition_file_ids" {
   type        = "list"
   default     = []

aws/kube-worker-general/worker.tf

@@ -21,7 +21,7 @@ data "null_data_source" "tags" {
 }
 
 resource "aws_autoscaling_group" "worker" {
-  name                 = "${var.name}-worker-general"
+  name                 = "${var.name}-worker-${var.worker_config["name"]}"
   desired_capacity     = "${var.worker_config["instance_count"]}"
   max_size             = "${var.worker_config["instance_count"] * 3}"
   min_size             = "${var.worker_config["instance_count"]}"
@@ -33,7 +33,7 @@ resource "aws_autoscaling_group" "worker" {
   tags = [
     {
       key                 = "Name"
-      value               = "${var.name}-worker-general"
+      value               = "${var.name}-worker-${var.worker_config["name"]}"
       propagate_at_launch = true
     },
     {
@@ -49,7 +49,7 @@ resource "aws_autoscaling_group" "worker" {
 resource "aws_launch_configuration" "worker" {
   instance_type = "${var.worker_config["ec2_type"]}"
   image_id      = "${data.aws_ami.coreos_ami.image_id}"
-  name_prefix   = "${var.name}-worker-general-"
+  name_prefix   = "${var.name}-worker-${var.worker_config["name"]}-"
 
   security_groups = [
     "${var.security_group_ids}",

aws/kube-worker-spot/ami.tf

@@ -0,0 +1,36 @@
+locals {
+  ami_owner = "595879546273"
+  arn       = "aws"
+
+  container_linux_channel = "stable"
+  container_linux_version = "latest"
+}
+
+module "container_linux" {
+  source = "../container_linux"
+
+  release_channel = "${local.container_linux_channel}"
+  release_version = "${local.container_linux_version}"
+}
+
+data "aws_ami" "coreos_ami" {
+  filter {
+    name   = "name"
+    values = ["CoreOS-${local.container_linux_channel}-${module.container_linux.version}-*"]
+  }
+
+  filter {
+    name   = "architecture"
+    values = ["x86_64"]
+  }
+
+  filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+
+  filter {
+    name   = "owner-id"
+    values = ["${local.ami_owner}"]
+  }
+}