Skip to content

Commit

Permalink
Merge branch 'master' into custom-gcr-project
Browse files Browse the repository at this point in the history
  • Loading branch information
morgante authored Oct 17, 2019
2 parents e34792c + aa9db52 commit 8c2e061
Show file tree
Hide file tree
Showing 68 changed files with 4,390 additions and 27 deletions.
10 changes: 10 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,14 @@ Extending the adopted spec, each change should have a link to its corresponding

## [Unreleased]

### Changed

* Made `region` variable optional for zonal clusters [#247]

### Added

* Added [private](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/tree/master/modules/private-cluster-update-variant) and [beta private](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/tree/master/modules/beta-private-cluster-update-variant) variants which allow node pools to be created before being destroyed. [#256]

## [v5.0.0] - 2019-09-25
v5.0.0 is a backwards-incompatible release. Please see the [upgrading guide](./docs/upgrading_to_v5.0.md).

Expand Down Expand Up @@ -196,6 +204,8 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
[v0.3.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.2.0...v0.3.0
[v0.2.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.1.0...v0.2.0

[#247]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/247
[#256]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/256
[#248]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/248
[#228]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/228
[#238]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/238
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no |
| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
| project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
| region | The region to host the cluster in (required) | string | n/a | yes |
| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
Expand Down
81 changes: 81 additions & 0 deletions autogen/cluster.tf
Original file line number Diff line number Diff line change
Expand Up @@ -219,14 +219,92 @@ resource "google_container_cluster" "primary" {
/******************************************
Create Container Cluster node pools
*****************************************/
{% if update_variant %}
locals {
force_node_pool_recreation_resources = [
"disk_size_gb",
"disk_type",
"accelerator_count",
"accelerator_type",
"local_ssd_count",
"machine_type",
"preemptible",
"service_account",
]
}

# This keepers list is based on the terraform google provider schemaNodeConfig
# resources where "ForceNew" is "true". schemaNodeConfig can be found in node_config.go at
# https://github.com/terraform-providers/terraform-provider-google/blob/master/google/node_config.go#L22
resource "random_id" "name" {
count = length(var.node_pools)
byte_length = 2
prefix = format("%s-", lookup(var.node_pools[count.index], "name"))
keepers = merge(
zipmap(
local.force_node_pool_recreation_resources,
[for keeper in local.force_node_pool_recreation_resources : lookup(var.node_pools[count.index], keeper, "")]
),
{
labels = join(",",
sort(
concat(
keys(var.node_pools_labels["all"]),
values(var.node_pools_labels["all"]),
keys(var.node_pools_labels[var.node_pools[count.index]["name"]]),
values(var.node_pools_labels[var.node_pools[count.index]["name"]])
)
)
)
},
{
metadata = join(",",
sort(
concat(
keys(var.node_pools_metadata["all"]),
values(var.node_pools_metadata["all"]),
keys(var.node_pools_metadata[var.node_pools[count.index]["name"]]),
values(var.node_pools_metadata[var.node_pools[count.index]["name"]])
)
)
)
},
{
oauth_scopes = join(",",
sort(
concat(
var.node_pools_oauth_scopes["all"],
var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]]
)
)
)
},
{
tags = join(",",
sort(
concat(
var.node_pools_tags["all"],
var.node_pools_tags[var.node_pools[count.index]["name"]]
)
)
)
}
)
}

{% endif %}
resource "google_container_node_pool" "pools" {
{% if beta_cluster %}
provider = google-beta
{% else %}
provider = google
{% endif %}
count = length(var.node_pools)
{% if update_variant %}
name = random_id.name.*.hex[count.index]
{% else %}
name = var.node_pools[count.index]["name"]
{% endif %}
project = var.project_id
location = local.location
cluster = google_container_cluster.primary.name
Expand Down Expand Up @@ -342,6 +420,9 @@ resource "google_container_node_pool" "pools" {

lifecycle {
ignore_changes = [initial_node_count]
{% if update_variant %}
create_before_destroy = true
{% endif %}
}

timeouts {
Expand Down
3 changes: 2 additions & 1 deletion autogen/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ data "google_compute_zones" "available" {
{% endif %}

project = var.project_id
region = var.region
region = local.region
}

resource "random_shuffle" "available_zones" {
Expand All @@ -38,6 +38,7 @@ resource "random_shuffle" "available_zones" {
locals {
// location
location = var.regional ? var.region : var.zones[0]
region = var.region == null ? join("-", slice(split("-", var.zones[0]), 0, 2)) : var.region
// for regional cluster - use var.zones if provided, use available otherwise, for zonal cluster use var.zones with first element extracted
node_locations = var.regional ? coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result)) : slice(var.zones, 1, length(var.zones))
// kuberentes version
Expand Down
2 changes: 1 addition & 1 deletion autogen/networks.tf
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,6 @@ data "google_compute_subnetwork" "gke_subnetwork" {
{% endif %}

name = var.subnetwork
region = var.region
region = local.region
project = local.network_project_id
}
3 changes: 2 additions & 1 deletion autogen/variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,8 @@ variable "regional" {

variable "region" {
type = string
description = "The region to host the cluster in (required)"
description = "The region to host the cluster in (optional if zonal cluster / required if regional)"
default = null
}

variable "zones" {
Expand Down
45 changes: 45 additions & 0 deletions examples/node_pool_update_variant/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
# Node Pool Cluster

This example illustrates how to create a cluster with multiple custom node-pool configurations with node labels, taints, and network tags.

<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
## Inputs

| Name | Description | Type | Default | Required |
|------|-------------|:----:|:-----:|:-----:|
| cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no |
| compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes |
| ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes |
| ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes |
| network | The VPC network to host the cluster in | string | n/a | yes |
| project\_id | The project ID to host the cluster in | string | n/a | yes |
| region | The region to host the cluster in | string | n/a | yes |
| subnetwork | The subnetwork to host the cluster in | string | n/a | yes |
| zones | The zone to host the cluster in (required if is a zonal cluster) | list(string) | n/a | yes |

## Outputs

| Name | Description |
|------|-------------|
| ca\_certificate | |
| client\_token | |
| cluster\_name | Cluster name |
| ip\_range\_pods | The secondary IP range used for pods |
| ip\_range\_services | The secondary IP range used for services |
| kubernetes\_endpoint | |
| location | |
| master\_kubernetes\_version | The master Kubernetes version |
| network | |
| project\_id | |
| region | |
| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
| subnetwork | |
| zones | List of zones in which the cluster resides |

<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

To provision this example, run the following from within this directory:
- `terraform init` to get the plugins
- `terraform plan` to see the infrastructure plan
- `terraform apply` to apply the infrastructure build
- `terraform destroy` to destroy the built infrastructure
17 changes: 17 additions & 0 deletions examples/node_pool_update_variant/data/shutdown-script.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
#!/bin/bash -e

# Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

kubectl --kubeconfig=/var/lib/kubelet/kubeconfig drain --force=true --ignore-daemonsets=true --delete-local-data "$HOSTNAME"
119 changes: 119 additions & 0 deletions examples/node_pool_update_variant/main.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
/**
* Copyright 2018 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

locals {
cluster_type = "node-pool-update-variant"
}

provider "google" {
version = "~> 2.12.0"
region = var.region
}

data "google_compute_subnetwork" "subnetwork" {
name = var.subnetwork
project = var.project_id
region = var.region
}

module "gke" {
source = "../../modules/private-cluster-update-variant"
project_id = var.project_id
name = "${local.cluster_type}-cluster${var.cluster_name_suffix}"
regional = false
region = var.region
zones = var.zones
network = var.network
subnetwork = var.subnetwork
ip_range_pods = var.ip_range_pods
ip_range_services = var.ip_range_services
create_service_account = false
service_account = var.compute_engine_service_account
enable_private_endpoint = true
enable_private_nodes = true
master_ipv4_cidr_block = "172.16.0.0/28"

master_authorized_networks_config = [
{
cidr_blocks = [
{
cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range
display_name = "VPC"
},
]
},
]

node_pools = [
{
name = "pool-01"
min_count = 1
max_count = 2
service_account = var.compute_engine_service_account
auto_upgrade = true
},
{
name = "pool-02"
machine_type = "n1-standard-2"
min_count = 1
max_count = 2
disk_size_gb = 30
disk_type = "pd-standard"
accelerator_count = 1
accelerator_type = "nvidia-tesla-p4"
image_type = "COS"
auto_repair = false
service_account = var.compute_engine_service_account
},
]

node_pools_oauth_scopes = {
all = []
pool-01 = []
pool-02 = []
}

node_pools_metadata = {
all = {}
pool-01 = {
shutdown-script = file("${path.module}/data/shutdown-script.sh")
}
pool-02 = {}
}

node_pools_labels = {
all = {
all-pools-example = true
}
pool-01 = {
pool-01-example = true
}
pool-02 = {}
}

node_pools_tags = {
all = [
"all-node-example",
]
pool-01 = [
"pool-01-example",
]
pool-02 = []
}
}

data "google_client_config" "default" {
}
35 changes: 35 additions & 0 deletions examples/node_pool_update_variant/outputs.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
/**
* Copyright 2018 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

output "kubernetes_endpoint" {
sensitive = true
value = module.gke.endpoint
}

output "client_token" {
sensitive = true
value = base64encode(data.google_client_config.default.access_token)
}

output "ca_certificate" {
value = module.gke.ca_certificate
}

output "service_account" {
description = "The service account to default running nodes as if not overridden in `node_pools`."
value = module.gke.service_account
}

Loading

0 comments on commit 8c2e061

Please sign in to comment.