feat: Add support for multiple registry projects (#815)

README.md

@@ -177,7 +177,8 @@ Then perform the following commands on the root folder:
 | project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes |
 | region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no |
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no |
-| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant\_registry\_access is true, storage.objectViewer role is assigned on this project. | `string` | `""` | no |
+| registry\_project\_id | Deprecated. Replaced by `registry_project_ids`. Still works for the purposes of backwards compatibility, but will be removed in a future version. | `string` | `""` | no |
+| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` role is assigned on these projects. | `list(string)` | `[]` | no |
 | release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | `string` | `null` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no |
 | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no |
@@ -282,7 +283,7 @@ following project roles:
 - roles/iam.serviceAccountUser
 - roles/resourcemanager.projectIamAdmin (only required if `service_account` is set to `create`)
 
-Additionally, if `service_account` is set to `create` and `grant_registry_access` is requested, the service account requires the following role on the `registry_project_id` project:
+Additionally, if `service_account` is set to `create` and `grant_registry_access` is requested, the service account requires the following role on the `registry_project_ids` projects:
 - roles/resourcemanager.projectIamAdmin
 
 ### Enable APIs

autogen/main/README.md

@@ -241,7 +241,7 @@ following project roles:
 - roles/iam.serviceAccountUser
 - roles/resourcemanager.projectIamAdmin (only required if `service_account` is set to `create`)
 
-Additionally, if `service_account` is set to `create` and `grant_registry_access` is requested, the service account requires the following role on the `registry_project_id` project:
+Additionally, if `service_account` is set to `create` and `grant_registry_access` is requested, the service account requires the following role on the `registry_project_ids` projects:
 - roles/resourcemanager.projectIamAdmin
 
 ### Enable APIs

autogen/main/sa.tf.tmpl

@@ -25,6 +25,12 @@ locals {
   )
   // if user set var.service_account it will be used even if var.create_service_account==true, so service account will be created but not used
   service_account = (var.service_account == "" || var.service_account == "create") && var.create_service_account ? local.service_account_list[0] : var.service_account
+
+  registry_projects_list = compact(
+    length(var.registry_project_ids) == 0 && var.registry_project_id == ""
+    ? [var.project_id]
+    : concat([var.registry_project_id], var.registry_project_ids)
+  )
 }
 
 resource "random_string" "cluster_service_account_suffix" {
@@ -70,15 +76,15 @@ resource "google_project_iam_member" "cluster_service_account-resourceMetadata-w
 }
 
 resource "google_project_iam_member" "cluster_service_account-gcr" {
-  count   = var.create_service_account && var.grant_registry_access ? 1 : 0
-  project = var.registry_project_id == "" ? var.project_id : var.registry_project_id
-  role    = "roles/storage.objectViewer"
-  member  = "serviceAccount:${google_service_account.cluster_service_account[0].email}"
+  for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : []
+  project  = each.key
+  role     = "roles/storage.objectViewer"
+  member   = "serviceAccount:${google_service_account.cluster_service_account[0].email}"
 }
 
 resource "google_project_iam_member" "cluster_service_account-artifact-registry" {
-  count   = var.create_service_account && var.grant_registry_access ? 1 : 0
-  project = var.registry_project_id == "" ? var.project_id : var.registry_project_id
-  role    = "roles/artifactregistry.reader"
-  member  = "serviceAccount:${google_service_account.cluster_service_account[0].email}"
+  for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : []
+  project  = each.key
+  role     = "roles/artifactregistry.reader"
+  member   = "serviceAccount:${google_service_account.cluster_service_account[0].email}"
 }

autogen/main/variables.tf.tmpl

@@ -360,10 +360,16 @@ variable "grant_registry_access" {
 
 variable "registry_project_id" {
   type        = string
-  description = "Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project."
+  description = "Deprecated. Replaced by `registry_project_ids`. Still works for the purposes of backwards compatibility, but will be removed in a future version."
   default     = ""
 }
 
+variable "registry_project_ids" {
+  type        = list(string)
+  description = "Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` role is assigned on these projects."
+  default     = []
+}
+
 variable "service_account" {
   type        = string
   description = "The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created."

autogen/safer-cluster/README.md

@@ -53,7 +53,8 @@ developers, which mostly just want to deploy and debug applications.
     own projects, so that they can be administered independently (e.g., dev cluster;
     production clusters; staging clusters should go in different projects.)
 
--   *A shared GCR project (`registry_project_id`):* all clusters can share the same GCR project.
+-   *Shared GCR projects (`registry_project_ids`):* all clusters can share the same
+    GCR projects.
 
     -   Easier to share images between environments. The same image could be
         progressively rolled-out in dev, staging, and then production.
@@ -93,7 +94,7 @@ The Safer Cluster setup relies on several service accounts:
 
 ```
 create_service_account = true
-registry_project_id = <the project id for your GCR project>
+registry_project_ids = [<the project id for your GCR project>]
 grant_registry_access = true
 ```
 

autogen/safer-cluster/main.tf.tmpl

@@ -101,6 +101,7 @@ module "gke" {
   create_service_account = var.compute_engine_service_account == "" ? true : false
   service_account        = var.compute_engine_service_account
   registry_project_id    = var.registry_project_id
+  registry_project_ids   = var.registry_project_ids
   grant_registry_access  = var.grant_registry_access
 
   // Basic Auth disabled

autogen/safer-cluster/variables.tf.tmpl

@@ -210,10 +210,16 @@ variable "grant_registry_access" {
 
 variable "registry_project_id" {
   type        = string
-  description = "Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project."
+  description = "Deprecated. Replaced by `registry_project_ids`. Still works for the purposes of backwards compatibility, but will be removed in a future version."
   default     = ""
 }
 
+variable "registry_project_ids" {
+  type        = list(string)
+  description = "Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` role is assigned on these projects."
+  default     = []
+}
+
 variable "cluster_resource_labels" {
   type        = map(string)
   description = "The GCE resource labels (a map of key/value pairs) to be applied to the cluster"

examples/workload_metadata_config/main.tf

@@ -51,7 +51,7 @@ module "gke" {
   ip_range_services       = var.ip_range_services
   create_service_account  = true
   grant_registry_access   = true
-  registry_project_id     = var.registry_project_id
+  registry_project_ids    = var.registry_project_ids
   enable_private_endpoint = true
   enable_private_nodes    = true
   master_ipv4_cidr_block  = "172.16.0.0/28"

examples/workload_metadata_config/outputs.tf

@@ -25,7 +25,8 @@ output "client_token" {
 }
 
 output "ca_certificate" {
-  value = module.gke.ca_certificate
+  sensitive = true
+  value     = module.gke.ca_certificate
 }
 
 output "service_account" {

examples/workload_metadata_config/variables.tf

@@ -48,6 +48,7 @@ variable "ip_range_services" {
   description = "The secondary ip range to use for services"
 }
 
-variable "registry_project_id" {
-  description = "Project name for the GCR registry"
+variable "registry_project_ids" {
+  description = "Project names for GCR registries"
+  type        = list(string)
 }

modules/beta-private-cluster-update-variant/README.md

@@ -232,7 +232,8 @@ Then perform the following commands on the root folder:
 | project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes |
 | region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no |
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no |
-| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant\_registry\_access is true, storage.objectViewer role is assigned on this project. | `string` | `""` | no |
+| registry\_project\_id | Deprecated. Replaced by `registry_project_ids`. Still works for the purposes of backwards compatibility, but will be removed in a future version. | `string` | `""` | no |
+| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` role is assigned on these projects. | `list(string)` | `[]` | no |
 | release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | `string` | `null` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no |
 | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no |
@@ -352,7 +353,7 @@ following project roles:
 - roles/iam.serviceAccountUser
 - roles/resourcemanager.projectIamAdmin (only required if `service_account` is set to `create`)
 
-Additionally, if `service_account` is set to `create` and `grant_registry_access` is requested, the service account requires the following role on the `registry_project_id` project:
+Additionally, if `service_account` is set to `create` and `grant_registry_access` is requested, the service account requires the following role on the `registry_project_ids` projects:
 - roles/resourcemanager.projectIamAdmin
 
 ### Enable APIs

modules/beta-private-cluster-update-variant/sa.tf

@@ -25,6 +25,12 @@ locals {
   )
   // if user set var.service_account it will be used even if var.create_service_account==true, so service account will be created but not used
   service_account = (var.service_account == "" || var.service_account == "create") && var.create_service_account ? local.service_account_list[0] : var.service_account
+
+  registry_projects_list = compact(
+    length(var.registry_project_ids) == 0 && var.registry_project_id == ""
+    ? [var.project_id]
+    : concat([var.registry_project_id], var.registry_project_ids)
+  )
 }
 
 resource "random_string" "cluster_service_account_suffix" {
@@ -70,15 +76,15 @@ resource "google_project_iam_member" "cluster_service_account-resourceMetadata-w
 }
 
 resource "google_project_iam_member" "cluster_service_account-gcr" {
-  count   = var.create_service_account && var.grant_registry_access ? 1 : 0
-  project = var.registry_project_id == "" ? var.project_id : var.registry_project_id
-  role    = "roles/storage.objectViewer"
-  member  = "serviceAccount:${google_service_account.cluster_service_account[0].email}"
+  for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : []
+  project  = each.key
+  role     = "roles/storage.objectViewer"
+  member   = "serviceAccount:${google_service_account.cluster_service_account[0].email}"
 }
 
 resource "google_project_iam_member" "cluster_service_account-artifact-registry" {
-  count   = var.create_service_account && var.grant_registry_access ? 1 : 0
-  project = var.registry_project_id == "" ? var.project_id : var.registry_project_id
-  role    = "roles/artifactregistry.reader"
-  member  = "serviceAccount:${google_service_account.cluster_service_account[0].email}"
+  for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : []
+  project  = each.key
+  role     = "roles/artifactregistry.reader"
+  member   = "serviceAccount:${google_service_account.cluster_service_account[0].email}"
 }

modules/beta-private-cluster-update-variant/variables.tf

@@ -348,10 +348,16 @@ variable "grant_registry_access" {
 
 variable "registry_project_id" {
   type        = string
-  description = "Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project."
+  description = "Deprecated. Replaced by `registry_project_ids`. Still works for the purposes of backwards compatibility, but will be removed in a future version."
   default     = ""
 }
 
+variable "registry_project_ids" {
+  type        = list(string)
+  description = "Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` role is assigned on these projects."
+  default     = []
+}
+
 variable "service_account" {
   type        = string
   description = "The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created."

modules/beta-private-cluster/README.md

@@ -210,7 +210,8 @@ Then perform the following commands on the root folder:
 | project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes |
 | region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no |
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no |
-| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant\_registry\_access is true, storage.objectViewer role is assigned on this project. | `string` | `""` | no |
+| registry\_project\_id | Deprecated. Replaced by `registry_project_ids`. Still works for the purposes of backwards compatibility, but will be removed in a future version. | `string` | `""` | no |
+| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` role is assigned on these projects. | `list(string)` | `[]` | no |
 | release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | `string` | `null` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no |
 | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no |
@@ -330,7 +331,7 @@ following project roles:
 - roles/iam.serviceAccountUser
 - roles/resourcemanager.projectIamAdmin (only required if `service_account` is set to `create`)
 
-Additionally, if `service_account` is set to `create` and `grant_registry_access` is requested, the service account requires the following role on the `registry_project_id` project:
+Additionally, if `service_account` is set to `create` and `grant_registry_access` is requested, the service account requires the following role on the `registry_project_ids` projects:
 - roles/resourcemanager.projectIamAdmin
 
 ### Enable APIs

modules/beta-private-cluster/sa.tf

@@ -25,6 +25,12 @@ locals {
   )
   // if user set var.service_account it will be used even if var.create_service_account==true, so service account will be created but not used
   service_account = (var.service_account == "" || var.service_account == "create") && var.create_service_account ? local.service_account_list[0] : var.service_account
+
+  registry_projects_list = compact(
+    length(var.registry_project_ids) == 0 && var.registry_project_id == ""
+    ? [var.project_id]
+    : concat([var.registry_project_id], var.registry_project_ids)
+  )
 }
 
 resource "random_string" "cluster_service_account_suffix" {
@@ -70,15 +76,15 @@ resource "google_project_iam_member" "cluster_service_account-resourceMetadata-w
 }
 
 resource "google_project_iam_member" "cluster_service_account-gcr" {
-  count   = var.create_service_account && var.grant_registry_access ? 1 : 0
-  project = var.registry_project_id == "" ? var.project_id : var.registry_project_id
-  role    = "roles/storage.objectViewer"
-  member  = "serviceAccount:${google_service_account.cluster_service_account[0].email}"
+  for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : []
+  project  = each.key
+  role     = "roles/storage.objectViewer"
+  member   = "serviceAccount:${google_service_account.cluster_service_account[0].email}"
 }
 
 resource "google_project_iam_member" "cluster_service_account-artifact-registry" {
-  count   = var.create_service_account && var.grant_registry_access ? 1 : 0
-  project = var.registry_project_id == "" ? var.project_id : var.registry_project_id
-  role    = "roles/artifactregistry.reader"
-  member  = "serviceAccount:${google_service_account.cluster_service_account[0].email}"
+  for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : []
+  project  = each.key
+  role     = "roles/artifactregistry.reader"
+  member   = "serviceAccount:${google_service_account.cluster_service_account[0].email}"
 }

modules/beta-private-cluster/variables.tf

@@ -348,10 +348,16 @@ variable "grant_registry_access" {
 
 variable "registry_project_id" {
   type        = string
-  description = "Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project."
+  description = "Deprecated. Replaced by `registry_project_ids`. Still works for the purposes of backwards compatibility, but will be removed in a future version."
   default     = ""
 }
 
+variable "registry_project_ids" {
+  type        = list(string)
+  description = "Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` role is assigned on these projects."
+  default     = []
+}
+
 variable "service_account" {
   type        = string
   description = "The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created."