Skip to content

Commit

Permalink
Merge pull request #108 from thefirstofthe300/ds/fix-version-skews
Browse files Browse the repository at this point in the history 
Fix issue with regional cluster roll outs causing version skews
  • Loading branch information
@aaron-lane
aaron-lane committed Mar 20, 2019
2 parents c105646 + 1d73e9d commit 567c586
Show file tree
Hide file tree
Showing 18 changed files with 101 additions and 35 deletions.
2 changes: 1 addition & 1 deletion auth.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,4 +31,4 @@ provider "kubernetes" {
host = "https://${local.cluster_endpoint}"
token = "${data.google_client_config.default.access_token}"
cluster_ca_certificate = "${base64decode(local.cluster_ca_certificate)}"
}
}
4 changes: 2 additions & 2 deletions autogen/cluster_regional.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ resource "google_container_cluster" "primary" {

network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}"
subnetwork = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}"
min_master_version = "${local.kubernetes_version}"
min_master_version = "${local.kubernetes_version_regional}"

logging_service = "${var.logging_service}"
monitoring_service = "${var.monitoring_service}"
Expand Down Expand Up @@ -104,7 +104,7 @@ resource "google_container_node_pool" "pools" {
project = "${var.project_id}"
region = "${var.region}"
cluster = "${var.name}"
version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version)}"
version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_regional)}"
initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}"

autoscaling {
Expand Down
4 changes: 2 additions & 2 deletions autogen/cluster_zonal.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ resource "google_container_cluster" "zonal_primary" {

network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}"
subnetwork = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}"
min_master_version = "${local.kubernetes_version}"
min_master_version = "${local.kubernetes_version_zonal}"

logging_service = "${var.logging_service}"
monitoring_service = "${var.monitoring_service}"
Expand Down Expand Up @@ -104,7 +104,7 @@ resource "google_container_node_pool" "zonal_pools" {
project = "${var.project_id}"
zone = "${var.zones[0]}"
cluster = "${var.name}"
version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version)}"
version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_zonal)}"
initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}"

autoscaling {
Expand Down
24 changes: 18 additions & 6 deletions autogen/main.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,12 @@ resource "random_shuffle" "available_zones" {
}

locals {
kubernetes_version = "${var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_node_version}"
node_version = "${var.node_version != "" ? var.node_version : local.kubernetes_version}"
custom_kube_dns_config = "${length(keys(var.stub_domains)) > 0 ? true : false}"
network_project_id = "${var.network_project_id != "" ? var.network_project_id : var.project_id}"
kubernetes_version_regional = "${var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version}"
kubernetes_version_zonal = "${var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version}"
node_version_regional = "${var.node_version != "" && var.regional ? var.node_version : local.kubernetes_version_regional}"
node_version_zonal = "${var.node_version != "" && !var.regional ? var.node_version : local.kubernetes_version_zonal}"
custom_kube_dns_config = "${length(keys(var.stub_domains)) > 0 ? true : false}"
network_project_id = "${var.network_project_id != "" ? var.network_project_id : var.project_id}"

cluster_type = "${var.regional ? "regional" : "zonal"}"

Expand Down Expand Up @@ -149,7 +151,17 @@ locals {
Get available container engine versions
*****************************************/
data "google_container_engine_versions" "region" {
provider = "{% if private_cluster %}google-beta{%else %}google{% endif %}"
zone = "${data.google_compute_zones.available.names[0]}"
provider = "google-beta"
region = "${var.region}"
project = "${var.project_id}"
}

data "google_container_engine_versions" "zone" {
provider = "google-beta"
// Work around to prevent a lack of zone declaration from causing regional cluster creation from erroring out due to error
//
// data.google_container_engine_versions.zone: Cannot determine zone: set in this resource, or set provider-level zone.
//
zone = "${var.zones[0] == "" ? data.google_compute_zones.available.names[0] : var.zones[0]}"
project = "${var.project_id}"
}
4 changes: 2 additions & 2 deletions cluster_regional.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ resource "google_container_cluster" "primary" {

network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}"
subnetwork = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}"
min_master_version = "${local.kubernetes_version}"
min_master_version = "${local.kubernetes_version_regional}"

logging_service = "${var.logging_service}"
monitoring_service = "${var.monitoring_service}"
Expand Down Expand Up @@ -97,7 +97,7 @@ resource "google_container_node_pool" "pools" {
project = "${var.project_id}"
region = "${var.region}"
cluster = "${var.name}"
version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version)}"
version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_regional)}"
initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}"

autoscaling {
Expand Down
6 changes: 3 additions & 3 deletions cluster_zonal.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ resource "google_container_cluster" "zonal_primary" {

network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}"
subnetwork = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}"
min_master_version = "${local.kubernetes_version}"
min_master_version = "${local.kubernetes_version_zonal}"

logging_service = "${var.logging_service}"
monitoring_service = "${var.monitoring_service}"
Expand Down Expand Up @@ -97,7 +97,7 @@ resource "google_container_node_pool" "zonal_pools" {
project = "${var.project_id}"
zone = "${var.zones[0]}"
cluster = "${var.name}"
version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version)}"
version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_zonal)}"
initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}"

autoscaling {
Expand All @@ -107,7 +107,7 @@ resource "google_container_node_pool" "zonal_pools" {

management {
auto_repair = "${lookup(var.node_pools[count.index], "auto_repair", true)}"
auto_upgrade = "${lookup(var.node_pools[count.index], "auto_upgrade", true)}"
auto_upgrade = "${lookup(var.node_pools[count.index], "auto_upgrade", false)}"
}

node_config {
Expand Down
5 changes: 5 additions & 0 deletions examples/deploy_service/main.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,11 @@ provider "google" {
version = "~> 1.20"
}

provider "google-beta" {
credentials = "${file(var.credentials_path)}"
region = "${var.region}"
}

provider "kubernetes" {
load_config_file = false
host = "https://${module.gke.endpoint}"
Expand Down
7 changes: 6 additions & 1 deletion examples/node_pool/main.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,11 @@ provider "google" {
region = "${var.region}"
}

provider "google-beta" {
credentials = "${file(var.credentials_path)}"
region = "${var.region}"
}

module "gke" {
source = "../../"
project_id = "${var.project_id}"
Expand All @@ -43,6 +48,7 @@ module "gke" {
min_count = 1
max_count = 2
service_account = "${var.compute_engine_service_account}"
auto_upgrade = true
},
{
name = "pool-02"
Expand All @@ -53,7 +59,6 @@ module "gke" {
disk_type = "pd-standard"
image_type = "COS"
auto_repair = false
auto_upgrade = false
service_account = "${var.compute_engine_service_account}"
},
]
Expand Down
5 changes: 5 additions & 0 deletions examples/shared_vpc/main.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,11 @@ provider "google" {
region = "${var.region}"
}

provider "google-beta" {
credentials = "${file(var.credentials_path)}"
region = "${var.region}"
}

module "gke" {
source = "../../"
project_id = "${var.project_id}"
Expand Down
5 changes: 5 additions & 0 deletions examples/simple_regional/main.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,11 @@ provider "google" {
region = "${var.region}"
}

provider "google-beta" {
credentials = "${file(var.credentials_path)}"
region = "${var.region}"
}

module "gke" {
source = "../../"
project_id = "${var.project_id}"
Expand Down
5 changes: 5 additions & 0 deletions examples/simple_zonal/main.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,11 @@ provider "google" {
region = "${var.region}"
}

provider "google-beta" {
credentials = "${file(var.credentials_path)}"
region = "${var.region}"
}

module "gke" {
source = "../../"
project_id = "${var.project_id}"
Expand Down
5 changes: 5 additions & 0 deletions examples/stub_domains/main.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,11 @@ provider "google" {
region = "${var.region}"
}

provider "google-beta" {
credentials = "${file(var.credentials_path)}"
region = "${var.region}"
}

module "gke" {
source = "../../"
project_id = "${var.project_id}"
Expand Down
26 changes: 19 additions & 7 deletions main.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,12 @@ resource "random_shuffle" "available_zones" {
}

locals {
kubernetes_version = "${var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_node_version}"
node_version = "${var.node_version != "" ? var.node_version : local.kubernetes_version}"
custom_kube_dns_config = "${length(keys(var.stub_domains)) > 0 ? true : false}"
network_project_id = "${var.network_project_id != "" ? var.network_project_id : var.project_id}"
kubernetes_version_regional = "${var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version}"
kubernetes_version_zonal = "${var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version}"
node_version_regional = "${var.node_version != "" && var.regional ? var.node_version : local.kubernetes_version_regional}"
node_version_zonal = "${var.node_version != "" && !var.regional ? var.node_version : local.kubernetes_version_zonal}"
custom_kube_dns_config = "${length(keys(var.stub_domains)) > 0 ? true : false}"
network_project_id = "${var.network_project_id != "" ? var.network_project_id : var.project_id}"

cluster_type = "${var.regional ? "regional" : "zonal"}"

Expand Down Expand Up @@ -149,7 +151,17 @@ locals {
Get available container engine versions
*****************************************/
data "google_container_engine_versions" "region" {
provider = "google"
zone = "${data.google_compute_zones.available.names[0]}"
provider = "google-beta"
region = "${var.region}"
project = "${var.project_id}"
}
}

data "google_container_engine_versions" "zone" {
// Work around to prevent a lack of zone declaration from causing regional cluster creation from erroring out due to error
//
// data.google_container_engine_versions.zone: Cannot determine zone: set in this resource, or set provider-level zone.
//
zone = "${var.zones[0] == "" ? data.google_compute_zones.available.names[0] : var.zones[0]}"

project = "${var.project_id}"
}
4 changes: 2 additions & 2 deletions modules/private-cluster/cluster_regional.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ resource "google_container_cluster" "primary" {

network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}"
subnetwork = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}"
min_master_version = "${local.kubernetes_version}"
min_master_version = "${local.kubernetes_version_regional}"

logging_service = "${var.logging_service}"
monitoring_service = "${var.monitoring_service}"
Expand Down Expand Up @@ -102,7 +102,7 @@ resource "google_container_node_pool" "pools" {
project = "${var.project_id}"
region = "${var.region}"
cluster = "${var.name}"
version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version)}"
version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_regional)}"
initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}"

autoscaling {
Expand Down
4 changes: 2 additions & 2 deletions modules/private-cluster/cluster_zonal.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ resource "google_container_cluster" "zonal_primary" {

network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}"
subnetwork = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}"
min_master_version = "${local.kubernetes_version}"
min_master_version = "${local.kubernetes_version_zonal}"

logging_service = "${var.logging_service}"
monitoring_service = "${var.monitoring_service}"
Expand Down Expand Up @@ -102,7 +102,7 @@ resource "google_container_node_pool" "zonal_pools" {
project = "${var.project_id}"
zone = "${var.zones[0]}"
cluster = "${var.name}"
version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version)}"
version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_zonal)}"
initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}"

autoscaling {
Expand Down
22 changes: 17 additions & 5 deletions modules/private-cluster/main.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,12 @@ resource "random_shuffle" "available_zones" {
}

locals {
kubernetes_version = "${var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_node_version}"
node_version = "${var.node_version != "" ? var.node_version : local.kubernetes_version}"
custom_kube_dns_config = "${length(keys(var.stub_domains)) > 0 ? true : false}"
network_project_id = "${var.network_project_id != "" ? var.network_project_id : var.project_id}"
kubernetes_version_regional = "${var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version}"
kubernetes_version_zonal = "${var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version}"
node_version_regional = "${var.node_version != "" && var.regional ? var.node_version : local.kubernetes_version_regional}"
node_version_zonal = "${var.node_version != "" && !var.regional ? var.node_version : local.kubernetes_version_zonal}"
custom_kube_dns_config = "${length(keys(var.stub_domains)) > 0 ? true : false}"
network_project_id = "${var.network_project_id != "" ? var.network_project_id : var.project_id}"

cluster_type = "${var.regional ? "regional" : "zonal"}"

Expand Down Expand Up @@ -150,6 +152,16 @@ locals {
*****************************************/
data "google_container_engine_versions" "region" {
provider = "google-beta"
zone = "${data.google_compute_zones.available.names[0]}"
region = "${var.region}"
project = "${var.project_id}"
}

data "google_container_engine_versions" "zone" {
// Work around to prevent a lack of zone declaration from causing regional cluster creation from erroring out due to error
//
// data.google_container_engine_versions.zone: Cannot determine zone: set in this resource, or set provider-level zone.
//
zone = "${var.zones[0] == "" ? data.google_compute_zones.available.names[0] : var.zones[0]}"

project = "${var.project_id}"
}
2 changes: 1 addition & 1 deletion modules/private-cluster/variables.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ variable "ip_range_pods" {
}

variable "ip_range_services" {
description = "The _name_ of the secondary subnet ip range to use for services"
description = "The _name_ of the secondary subnet range to use for services"
}

variable "remove_default_node_pool" {
Expand Down
2 changes: 1 addition & 1 deletion variables.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ variable "ip_range_pods" {
}

variable "ip_range_services" {
description = "The _name_ of the secondary subnet ip range to use for services"
description = "The _name_ of the secondary subnet range to use for services"
}

variable "remove_default_node_pool" {
Expand Down

0 comments on commit 567c586

Please sign in to comment.