feat!: Set the provided SA when creating autopilot clusters (#1495)

Co-authored-by: Bharath KKB <bharathkrishnakb@gmail.com>

autogen/main/cluster.tf.tmpl

@@ -150,6 +150,17 @@ resource "google_container_cluster" "primary" {
     }
   }
   {% endif %}
+  {% if autopilot_cluster == true %}
+  cluster_autoscaling {
+    dynamic "auto_provisioning_defaults" {
+      for_each = var.create_service_account ? [1] : []
+
+      content {
+        service_account = local.service_account
+      }
+    }
+  }
+  {% endif %}
   vertical_pod_autoscaling {
     enabled = var.enable_vertical_pod_autoscaling
   }

autogen/main/versions.tf.tmpl

@@ -24,7 +24,7 @@ terraform {
   required_providers {
     google-beta = {
       source  = "hashicorp/google-beta"
-      version = ">= 4.42.0, < 5.0"
+      version = ">= 4.44.0, < 5.0"
     }
     kubernetes = {
       source  = "hashicorp/kubernetes"
@@ -38,7 +38,7 @@ terraform {
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = ">= 4.36.0, < 5.0"
+      version = ">= 4.44.0, < 5.0"
     }
     kubernetes = {
       source  = "hashicorp/kubernetes"

build/int.cloudbuild.yaml

@@ -401,9 +401,26 @@ steps:
     - verify private-zonal-with-networking
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'cft test run TestPrivateZonalWithNetworking --stage teardown --verbose --test-dir test/integration']
-
-
-
+- id: init simple-autopilot-private-non-default-sa
+  waitFor:
+    - prepare
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestSimpleAutopilotPrivateNonDefaultSA --stage init --verbose']
+- id: apply simple-autopilot-private-non-default-sa
+  waitFor:
+    - init simple-autopilot-private-non-default-sa
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestSimpleAutopilotPrivateNonDefaultSA --stage apply --verbose']
+- id: verify simple-autopilot-private-non-default-sa
+  waitFor:
+    - apply simple-autopilot-private-non-default-sa
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestSimpleAutopilotPrivateNonDefaultSA --stage verify --verbose']
+- id: teardown simple-autopilot-private-non-default-sa
+  waitFor:
+    - verify simple-autopilot-private-non-default-sa
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestSimpleAutopilotPrivateNonDefaultSA --stage teardown --verbose']
 tags:
 - 'ci'
 - 'integration'

examples/simple_autopilot_private_non_default_sa/README.md

@@ -0,0 +1,33 @@
+# Simple Regional Autopilot Cluster
+
+This example illustrates how to create a simple autopilot cluster with beta features and
+not using the default service account.
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| project\_id | The project ID to host the cluster in | `any` | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| cluster\_name | Cluster name |
+| kubernetes\_endpoint | The cluster endpoint |
+| location | n/a |
+| master\_kubernetes\_version | Kubernetes version of the master |
+| network\_name | The name of the VPC being created |
+| region | The region in which the cluster resides |
+| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
+| subnet\_names | The names of the subnet being created |
+| zones | List of zones in which the cluster resides |
+
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+
+To provision this example, run the following from within this directory:
+- `terraform init` to get the plugins
+- `terraform plan` to see the infrastructure plan
+- `terraform apply` to apply the infrastructure build
+- `terraform destroy` to destroy the built infrastructure

examples/simple_autopilot_private_non_default_sa/main.tf

@@ -0,0 +1,59 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+locals {
+  cluster_type           = "simple-ap-private-non-default-sa"
+  network_name           = "${local.cluster_type}-network"
+  subnet_name            = "${local.cluster_type}-subnet"
+  master_auth_subnetwork = "${local.cluster_type}-master-subnet"
+  pods_range_name        = "ip-range-pods-${local.cluster_type}"
+  svc_range_name         = "ip-range-svc-${local.cluster_type}"
+  subnet_names           = [for subnet_self_link in module.gcp-network.subnets_self_links : split("/", subnet_self_link)[length(split("/", subnet_self_link)) - 1]]
+}
+
+
+data "google_client_config" "default" {}
+
+provider "kubernetes" {
+  host                   = "https://${module.gke.endpoint}"
+  token                  = data.google_client_config.default.access_token
+  cluster_ca_certificate = base64decode(module.gke.ca_certificate)
+}
+
+module "gke" {
+  source                          = "../../modules/beta-autopilot-private-cluster/"
+  project_id                      = var.project_id
+  name                            = "${local.cluster_type}-cluster"
+  regional                        = true
+  region                          = "us-central1"
+  network                         = module.gcp-network.network_name
+  subnetwork                      = local.subnet_names[index(module.gcp-network.subnets_names, local.subnet_name)]
+  ip_range_pods                   = local.pods_range_name
+  ip_range_services               = local.svc_range_name
+  release_channel                 = "REGULAR"
+  enable_vertical_pod_autoscaling = true
+  enable_private_endpoint         = true
+  enable_private_nodes            = true
+  master_ipv4_cidr_block          = "172.16.0.0/28"
+  datapath_provider               = "ADVANCED_DATAPATH"
+
+  master_authorized_networks = [
+    {
+      cidr_block   = "10.60.0.0/17"
+      display_name = "VPC"
+    },
+  ]
+}

examples/simple_autopilot_private_non_default_sa/network.tf

@@ -0,0 +1,50 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module "gcp-network" {
+  source  = "terraform-google-modules/network/google"
+  version = ">= 4.0.1"
+
+  project_id   = var.project_id
+  network_name = local.network_name
+
+  subnets = [
+    {
+      subnet_name           = local.subnet_name
+      subnet_ip             = "10.0.0.0/17"
+      subnet_region         = "us-central1"
+      subnet_private_access = true
+    },
+    {
+      subnet_name   = local.master_auth_subnetwork
+      subnet_ip     = "10.60.0.0/17"
+      subnet_region = "us-central1"
+    },
+  ]
+
+  secondary_ranges = {
+    (local.subnet_name) = [
+      {
+        range_name    = local.pods_range_name
+        ip_cidr_range = "192.168.0.0/18"
+      },
+      {
+        range_name    = local.svc_range_name
+        ip_cidr_range = "192.168.64.0/18"
+      },
+    ]
+  }
+}

examples/simple_autopilot_private_non_default_sa/outputs.tf

@@ -0,0 +1,60 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "kubernetes_endpoint" {
+  description = "The cluster endpoint"
+  sensitive   = true
+  value       = module.gke.endpoint
+}
+
+output "cluster_name" {
+  description = "Cluster name"
+  value       = module.gke.name
+}
+
+output "location" {
+  value = module.gke.location
+}
+
+output "master_kubernetes_version" {
+  description = "Kubernetes version of the master"
+  value       = module.gke.master_version
+}
+
+output "service_account" {
+  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  value       = module.gke.service_account
+}
+
+output "network_name" {
+  description = "The name of the VPC being created"
+  value       = module.gcp-network.network_name
+}
+
+output "subnet_names" {
+  description = "The names of the subnet being created"
+  value       = module.gcp-network.subnets_names
+}
+
+output "region" {
+  description = "The region in which the cluster resides"
+  value       = module.gke.region
+}
+
+output "zones" {
+  description = "List of zones in which the cluster resides"
+  value       = module.gke.zones
+}

examples/simple_autopilot_private_non_default_sa/variables.tf

@@ -0,0 +1,19 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "project_id" {
+  description = "The project ID to host the cluster in"
+}

examples/simple_autopilot_private_non_default_sa/versions.tf

@@ -0,0 +1,28 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+terraform {
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "~> 4.0"
+    }
+    kubernetes = {
+      source = "hashicorp/kubernetes"
+    }
+  }
+  required_version = ">= 0.13"
+}

modules/beta-autopilot-private-cluster/cluster.tf

@@ -71,6 +71,15 @@ resource "google_container_cluster" "primary" {
       }
     }
   }
+  cluster_autoscaling {
+    dynamic "auto_provisioning_defaults" {
+      for_each = var.create_service_account ? [1] : []
+
+      content {
+        service_account = local.service_account
+      }
+    }
+  }
   vertical_pod_autoscaling {
     enabled = var.enable_vertical_pod_autoscaling
   }

modules/beta-autopilot-private-cluster/versions.tf

@@ -21,7 +21,7 @@ terraform {
   required_providers {
     google-beta = {
       source  = "hashicorp/google-beta"
-      version = ">= 4.42.0, < 5.0"
+      version = ">= 4.44.0, < 5.0"
     }
     kubernetes = {
       source  = "hashicorp/kubernetes"

modules/beta-autopilot-public-cluster/cluster.tf

@@ -71,6 +71,15 @@ resource "google_container_cluster" "primary" {
       }
     }
   }
+  cluster_autoscaling {
+    dynamic "auto_provisioning_defaults" {
+      for_each = var.create_service_account ? [1] : []
+
+      content {
+        service_account = local.service_account
+      }
+    }
+  }
   vertical_pod_autoscaling {
     enabled = var.enable_vertical_pod_autoscaling
   }

modules/beta-autopilot-public-cluster/versions.tf

@@ -21,7 +21,7 @@ terraform {
   required_providers {
     google-beta = {
       source  = "hashicorp/google-beta"
-      version = ">= 4.42.0, < 5.0"
+      version = ">= 4.44.0, < 5.0"
     }
     kubernetes = {
       source  = "hashicorp/kubernetes"

modules/beta-private-cluster-update-variant/versions.tf

@@ -21,7 +21,7 @@ terraform {
   required_providers {
     google-beta = {
       source  = "hashicorp/google-beta"
-      version = ">= 4.42.0, < 5.0"
+      version = ">= 4.44.0, < 5.0"
     }
     kubernetes = {
       source  = "hashicorp/kubernetes"

modules/beta-private-cluster/versions.tf

@@ -21,7 +21,7 @@ terraform {
   required_providers {
     google-beta = {
       source  = "hashicorp/google-beta"
-      version = ">= 4.42.0, < 5.0"
+      version = ">= 4.44.0, < 5.0"
     }
     kubernetes = {
       source  = "hashicorp/kubernetes"

modules/beta-public-cluster-update-variant/versions.tf

@@ -21,7 +21,7 @@ terraform {
   required_providers {
     google-beta = {
       source  = "hashicorp/google-beta"
-      version = ">= 4.42.0, < 5.0"
+      version = ">= 4.44.0, < 5.0"
     }
     kubernetes = {
       source  = "hashicorp/kubernetes"