Skip to content

Commit

Permalink
modules/aws/bootstrap: Pull AWS bootstrap setup into a module
Browse files Browse the repository at this point in the history
This will make it easier to move into the existing infra step.

The module source syntax used in the README is documented in [1,2,3],
and means "the modules/aws/ami subdirectory of the
github.com/openshift/installer repository cloned over HTTPS", etc.

I don't think I should need the wrapping brackets in:

  vpc_security_group_ids = ["${var.vpc_security_group_ids}"]

but without it I get [4]:

  Error: module.bootstrap.aws_instance.bootstrap: vpc_security_group_ids: should be a list

The explicit brackets match our approach in the master and worker
modules though, so they shouldn't break anything.  It sounds like
Terraform still has a few problems with remembering type information
[5], and that may be what's going on here.

I've simplified the tagging a bit, keeping the extra tags unification
outside the module.  I tried dropping the kubernetes.io/cluster/ tag
completely, but it lead to [6]:

  Sep 12 05:08:08 ip-10-0-6-58 hyperkube[4122]: E0912 05:08:08.075211    4122 tags.go:94] Tag "KubernetesCluster" nor "kubernetes.io/cluster/..." not found; Kubernetes may behave unexpectedly.
  Sep 12 05:08:08 ip-10-0-6-58 hyperkube[4122]: F0912 05:08:08.075258    4122 server.go:262] failed to run Kubelet: could not init cloud provider "aws": AWS cloud failed to find ClusterID

The backing code for that is [7,8,9].  From [9], you can see that only
the tag on the instance matters, so I've dropped
kubernetes.io/cluster/... from volume_tags.  Going forward, we may
move to configuring this directly instead of relying on the tag-based
initialization.

[1]: https://www.terraform.io/docs/configuration/modules.html#source
[2]: https://www.terraform.io/docs/modules/sources.html#github
[3]: https://www.terraform.io/docs/modules/sources.html#modules-in-package-sub-directories
[4]: https://storage.googleapis.com/origin-ci-test/pr-logs/pull/openshift_installer/217/pull-ci-openshift-installer-e2e-aws/47/build-log.txt
[5]: hashicorp/terraform#16916 (comment)
[6]: openshift#217
[7]: https://github.com/kubernetes/kubernetes/blob/v1.11.3/pkg/cloudprovider/providers/aws/tags.go#L30-L34
[8]: https://github.com/kubernetes/kubernetes/blob/v1.11.3/pkg/cloudprovider/providers/aws/tags.go#L100-L126
[9]: https://github.com/kubernetes/kubernetes/blob/v1.11.3/pkg/cloudprovider/providers/aws/aws.go#L1126-L1132
  • Loading branch information
wking committed Sep 12, 2018
1 parent 69406be commit 8a37f72
Show file tree
Hide file tree
Showing 4 changed files with 264 additions and 130 deletions.
51 changes: 51 additions & 0 deletions modules/aws/bootstrap/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
# Bootstrap Module

This [Terraform][] [module][] manages [AWS][] resources only needed during cluster bootstrapping.
It uses [implicit provider inheritance][implicit-provider-inheritance] to access the [AWS provider][AWS-provider].

## Example

Set up a `main.tf` with:

```hcl
provider "aws" {
region = "us-east-1"
}
resource "aws_s3_bucket" "example" {
}
resource "aws_vpc" "example" {
cidr_block = "10.0.0.0/16"
enable_dns_hostnames = true
enable_dns_support = true
}
resource "aws_subnet" "example" {
vpc_id = "${aws_vpc.example.id}"
cidr_block = "${aws_vpc.example.cidr_block}"
}
module "bootstrap" {
source = "github.com/openshift/installer//modules/aws/bootstrap"
ami = "ami-07307c397daf4d02e"
bucket = "${aws_s3_bucket.example.id}"
cluster_name = "my-cluster"
ignition = "{\"ignition\": {\"version\": \"2.2.0\"}}",
subnet_id = "${aws_subnet.example.id}"
}
```

Then run:

```console
$ terraform init
$ terraform plan
```

[AWS]: https://aws.amazon.com/
[AWS-provider]: https://www.terraform.io/docs/providers/aws/
[implicit-provider-inheritance]: https://www.terraform.io/docs/modules/usage.html#implicit-provider-inheritance
[module]: https://www.terraform.io/docs/modules/
[Terraform]: https://www.terraform.io/
127 changes: 127 additions & 0 deletions modules/aws/bootstrap/main.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
resource "aws_s3_bucket_object" "ignition" {
bucket = "${var.bucket}"
key = "bootstrap.ign"
content = "${var.ignition}"
acl = "private"

server_side_encryption = "AES256"

tags = "${var.tags}"

lifecycle {
ignore_changes = ["*"]
}
}

data "ignition_config" "redirect" {
replace {
source = "s3://${var.bucket}/bootstrap.ign"
}
}

resource "aws_iam_instance_profile" "bootstrap" {
name = "${var.cluster_name}-bootstrap-profile"

role = "${var.iam_role == "" ?
join("|", aws_iam_role.bootstrap.*.name) :
join("|", data.aws_iam_role.bootstrap.*.name)
}"
}

data "aws_iam_role" "bootstrap" {
count = "${var.iam_role == "" ? 0 : 1}"
name = "${var.iam_role}"
}

resource "aws_iam_role" "bootstrap" {
count = "${var.iam_role == "" ? 1 : 0}"
name = "${var.cluster_name}-bootstrap-role"
path = "/"

assume_role_policy = <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Action": "sts:AssumeRole",
"Principal": {
"Service": "ec2.amazonaws.com"
},
"Effect": "Allow",
"Sid": ""
}
]
}
EOF
}

resource "aws_iam_role_policy" "bootstrap" {
count = "${var.iam_role == "" ? 1 : 0}"
name = "${var.cluster_name}-bootstrap-policy"
role = "${aws_iam_role.bootstrap.id}"

policy = <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": "ec2:Describe*",
"Resource": "*"
},
{
"Effect": "Allow",
"Action": "ec2:AttachVolume",
"Resource": "*"
},
{
"Effect": "Allow",
"Action": "ec2:DetachVolume",
"Resource": "*"
},
{
"Action" : [
"s3:GetObject"
],
"Resource": "arn:aws:s3:::*",
"Effect": "Allow"
}
]
}
EOF
}

resource "aws_instance" "bootstrap" {
ami = "${var.ami}"

iam_instance_profile = "${aws_iam_instance_profile.bootstrap.name}"
instance_type = "${var.instance_type}"
subnet_id = "${var.subnet_id}"
user_data = "${data.ignition_config.redirect.rendered}"
vpc_security_group_ids = ["${var.vpc_security_group_ids}"]
associate_public_ip_address = "${var.associate_public_ip_address}"

lifecycle {
# Ignore changes in the AMI which force recreation of the resource. This
# avoids accidental deletion of nodes whenever a new OS release comes out.
ignore_changes = ["ami"]
}

tags = "${merge(map(
"kubernetes.io/cluster/${var.cluster_name}", "owned",
), var.tags)}"

root_block_device {
volume_type = "${var.volume_type}"
volume_size = "${var.volume_size}"
iops = "${var.volume_type == "io1" ? var.volume_iops : 0}"
}

volume_tags = "${var.tags}"
}

resource "aws_elb_attachment" "bootstrap" {
count = "${length(var.elbs)}"
elb = "${var.elbs[count.index]}"
instance = "${aws_instance.bootstrap.id}"
}
77 changes: 77 additions & 0 deletions modules/aws/bootstrap/variables.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
variable "ami" {
type = "string"
description = "The AMI ID for the bootstrap node."
}

variable "associate_public_ip_address" {
default = false
description = "If set to true, public-facing ingress resources are created."
}

variable "bucket" {
type = "string"
description = "The S3 bucket name for bootstrap ignition file."
}

variable "cluster_name" {
type = "string"
description = "The name of the cluster."
}

variable "elbs" {
type = "list"
default = []
description = "Elastic load balancer IDs to attach to the bootstrap node."
}

variable "iam_role" {
type = "string"
default = ""
description = "The name of the IAM role to assign to the bootstrap node."
}

variable "ignition" {
type = "string"
description = "The content of the bootstrap ignition file."
}

variable "instance_type" {
type = "string"
default = "t2.medium"
description = "The EC2 instance type for the bootstrap node."
}

variable "subnet_id" {
type = "string"
description = "The subnet ID for the bootstrap node."
}

variable "tags" {
type = "map"
default = {}
description = "AWS tags to be applied to created resources."
}

variable "volume_iops" {
type = "string"
default = "100"
description = "The amount of IOPS to provision for the disk."
}

variable "volume_size" {
type = "string"
default = "30"
description = "The volume size (in gibibytes) for the bootstrap node's root volume."
}

variable "volume_type" {
type = "string"
default = "gp2"
description = "The volume type for the bootstrap node's root volume."
}

variable "vpc_security_group_ids" {
type = "list"
default = []
description = "VPC security group IDs for the bootstrap node."
}
Loading

0 comments on commit 8a37f72

Please sign in to comment.