feat: add single project example

2-multitenant/modules/env_baseline/main.tf

@@ -21,6 +21,8 @@ locals {
   cluster_project_id    = data.google_project.eab_cluster_project.project_id
   available_cidr_ranges = var.master_ipv4_cidr_blocks
 
+  subnets = { for idx, v in var.cluster_subnetworks : idx => v }
+
   subnets_to_cidr = {
     for idx, subnet_key in keys(data.google_compute_subnetwork.default) : subnet_key => local.available_cidr_ranges[idx]
   }
@@ -83,6 +85,7 @@ module "cloud_armor" {
   type                                 = "CLOUD_ARMOR"
   layer_7_ddos_defense_enable          = true
   layer_7_ddos_defense_rule_visibility = "STANDARD"
+  user_ip_request_headers              = []
 
   pre_configured_rules = {
     "sqli_sensitivity_level_1" = {
@@ -104,10 +107,50 @@ module "cloud_armor" {
 
 // Retrieve the subnetworks
 data "google_compute_subnetwork" "default" {
-  for_each  = { for value in var.cluster_subnetworks : regex(local.subnetworks_re, value)[0] => value }
+  for_each  = local.subnets
   self_link = each.value
 }
 
+resource "google_project_service_identity" "gke_identity_cluster_project" {
+  provider   = google-beta
+  project    = local.cluster_project_id
+  service    = "gkehub.googleapis.com"
+  depends_on = [module.eab_cluster_project]
+}
+
+resource "google_project_service_identity" "mcsd_cluster_project" {
+  provider   = google-beta
+  project    = local.cluster_project_id
+  service    = "multiclusterservicediscovery.googleapis.com"
+  depends_on = [module.eab_cluster_project]
+}
+
+resource "google_project_iam_member" "gke_service_agent" {
+  project    = local.cluster_project_id
+  role       = "roles/gkehub.serviceAgent"
+  member     = google_project_service_identity.gke_identity_cluster_project.member
+  depends_on = [module.eab_cluster_project]
+}
+
+resource "google_project_service_identity" "fleet_meshconfig_sa" {
+  provider = google-beta
+  project  = local.cluster_project_id
+  service  = "meshconfig.googleapis.com"
+}
+
+resource "google_project_iam_member" "servicemesh_service_agent" {
+  project    = local.cluster_project_id
+  role       = "roles/meshconfig.serviceAgent"
+  member     = google_project_service_identity.fleet_meshconfig_sa.member
+  depends_on = [module.eab_cluster_project, google_project_service_identity.fleet_meshconfig_sa]
+}
+
+resource "google_project_iam_member" "multiclusterdiscovery_service_agent" {
+  project = local.cluster_project_id
+  role    = "roles/multiclusterservicediscovery.serviceAgent"
+  member  = google_project_service_identity.mcsd_cluster_project.member
+}
+
 module "gke-standard" {
   source  = "terraform-google-modules/kubernetes-engine/google//modules/beta-private-cluster"
   version = "~> 33.0"
@@ -120,7 +163,7 @@ module "gke-standard" {
   region                 = each.value.region
   network_project_id     = regex(local.projects_re, each.value.id)[0]
   network                = regex(local.networks_re, each.value.network)[0]
-  subnetwork             = each.value.name
+  subnetwork             = regex(local.subnetworks_re, local.subnets[each.key])[0]
   ip_range_pods          = each.value.secondary_ip_range[0].range_name
   ip_range_services      = each.value.secondary_ip_range[1].range_name
   release_channel        = var.cluster_release_channel
@@ -175,13 +218,18 @@ module "gke-standard" {
   ]
 
   depends_on = [
-    module.eab_cluster_project
+    module.eab_cluster_project,
+    google_project_iam_member.gke_service_agent,
+    google_project_iam_member.servicemesh_service_agent,
+    google_project_iam_member.multiclusterdiscovery_service_agent
   ]
 
   // Private Cluster Configuration
   enable_private_nodes    = true
   enable_private_endpoint = true
 
+  fleet_project_grant_service_agent = true
+
   deletion_protection = false # set to true to prevent the module from deleting the cluster on destroy
 }
 
@@ -218,12 +266,23 @@ module "gke-autopilot" {
   }
 
   depends_on = [
-    module.eab_cluster_project
+    module.eab_cluster_project,
+    google_project_iam_member.gke_service_agent,
+    google_project_iam_member.servicemesh_service_agent,
+    google_project_iam_member.multiclusterdiscovery_service_agent
   ]
 
   // Private Cluster Configuration
   enable_private_nodes    = true
   enable_private_endpoint = true
 
+  fleet_project_grant_service_agent = true
+
   deletion_protection = false # set to true to prevent the module from deleting the cluster on destroy
 }
+
+resource "time_sleep" "wait_service_cleanup" {
+  depends_on = [module.gke-autopilot.name, module.gke-standard.name]
+
+  destroy_duration = "300s"
+}

2-multitenant/modules/env_baseline/outputs.tf

@@ -31,6 +31,8 @@ output "cluster_membership_ids" {
 output "cluster_project_id" {
   description = "Cluster Project ID"
   value       = data.google_project.eab_cluster_project.project_id
+
+  depends_on = [module.gke-standard, module.gke-autopilot]
 }
 
 output "network_project_id" {

3-fleetscope/envs/development/main.tf

@@ -18,10 +18,10 @@ locals {
   env = "development"
 }
 
-import {
-  id = "projects/${local.cluster_project_id}/locations/global/features/fleetobservability"
-  to = module.env.google_gke_hub_feature.fleet-o11y
-}
+# import {
+#   id = "projects/${local.cluster_project_id}/locations/global/features/fleetobservability"
+#   to = module.env.google_gke_hub_feature.fleet-o11y
+# }
 
 module "env" {
   source = "../../modules/env_baseline"

3-fleetscope/envs/nonproduction/main.tf

@@ -18,10 +18,10 @@ locals {
   env = "nonproduction"
 }
 
-import {
-  id = "projects/${local.cluster_project_id}/locations/global/features/fleetobservability"
-  to = module.env.google_gke_hub_feature.fleet-o11y
-}
+# import {
+#   id = "projects/${local.cluster_project_id}/locations/global/features/fleetobservability"
+#   to = module.env.google_gke_hub_feature.fleet-o11y
+# }
 
 module "env" {
   source = "../../modules/env_baseline"

3-fleetscope/envs/production/main.tf

@@ -18,10 +18,10 @@ locals {
   env = "production"
 }
 
-import {
-  id = "projects/${local.cluster_project_id}/locations/global/features/fleetobservability"
-  to = module.env.google_gke_hub_feature.fleet-o11y
-}
+# import {
+#   id = "projects/${local.cluster_project_id}/locations/global/features/fleetobservability"
+#   to = module.env.google_gke_hub_feature.fleet-o11y
+# }
 
 module "env" {
   source = "../../modules/env_baseline"

3-fleetscope/modules/env_baseline/acm.tf

@@ -14,6 +14,10 @@
  * limitations under the License.
  */
 
+locals {
+  cluster_membership_ids = { for k, v in var.cluster_membership_ids : k => v }
+}
+
 data "google_project" "cluster_project" {
   project_id = var.cluster_project_id
 }
@@ -36,13 +40,11 @@ resource "google_project_iam_member" "root_reconciler" {
   member  = "serviceAccount:${google_service_account.root_reconciler.email}"
 }
 
-resource "google_service_account_iam_binding" "workload_identity" {
+resource "google_service_account_iam_member" "workload_identity" {
   service_account_id = google_service_account.root_reconciler.name
-  role               = "roles/iam.workloadIdentityUser"
 
-  members = [
-    "serviceAccount:${var.cluster_project_id}.svc.id.goog[config-management-system/root-reconciler]",
-  ]
+  role   = "roles/iam.workloadIdentityUser"
+  member = "serviceAccount:${var.cluster_project_id}.svc.id.goog[config-management-system/root-reconciler]"
 }
 
 resource "google_gke_hub_feature" "acm_feature" {
@@ -52,14 +54,14 @@ resource "google_gke_hub_feature" "acm_feature" {
 }
 
 resource "google_gke_hub_feature_membership" "acm_feature_member" {
-  for_each = toset(var.cluster_membership_ids)
+  for_each = local.cluster_membership_ids
 
   project  = var.cluster_project_id
   location = "global"
 
   feature             = google_gke_hub_feature.acm_feature.name
-  membership          = regex(local.membership_re, each.key)[2]
-  membership_location = regex(local.membership_re, each.key)[1]
+  membership          = regex(local.membership_re, each.value)[2]
+  membership_location = regex(local.membership_re, each.value)[1]
 
   configmanagement {
     version = "1.19.0"

3-fleetscope/modules/env_baseline/asm.tf

@@ -33,11 +33,11 @@ resource "google_gke_hub_feature_membership" "mesh_feature_member" {
   project  = var.fleet_project_id
   location = "global"
 
-  for_each = toset(var.cluster_membership_ids)
+  for_each = local.cluster_membership_ids
 
   feature             = google_gke_hub_feature.mesh_feature.name
-  membership          = regex(local.membership_re, each.key)[2]
-  membership_location = regex(local.membership_re, each.key)[1]
+  membership          = regex(local.membership_re, each.value)[2]
+  membership_location = regex(local.membership_re, each.value)[1]
 
   mesh {
     management = "MANAGEMENT_AUTOMATIC"

3-fleetscope/modules/env_baseline/log.tf

@@ -14,28 +14,28 @@
  * limitations under the License.
  */
 
-resource "google_gke_hub_feature" "fleet-o11y" {
-  name     = "fleetobservability"
-  project  = var.fleet_project_id
-  location = "global"
-  spec {
-    fleetobservability {
-      logging_config {
-        default_config {
-          mode = "COPY"
-        }
-        fleet_scope_logs_config {
-          mode = "MOVE"
-        }
-      }
-    }
-  }
+# resource "google_gke_hub_feature" "fleet-o11y" {
+#   name     = "fleetobservability"
+#   project  = var.fleet_project_id
+#   location = "global"
+#   spec {
+#     fleetobservability {
+#       logging_config {
+#         default_config {
+#           mode = "COPY"
+#         }
+#         fleet_scope_logs_config {
+#           mode = "MOVE"
+#         }
+#       }
+#     }
+#   }
 
-  depends_on = [
-    google_gke_hub_feature.mesh_feature,
-    google_project_iam_member.fleet_logging_viewaccessor
-  ]
-}
+#   depends_on = [
+#     google_gke_hub_feature.mesh_feature,
+#     google_project_iam_member.fleet_logging_viewaccessor
+#   ]
+# }
 
 resource "google_project_iam_member" "fleet_logging_viewaccessor" {
   for_each = var.namespace_ids

3-fleetscope/modules/env_baseline/main.tf

@@ -16,8 +16,9 @@
 
 locals {
   membership_re = "//gkehub.googleapis.com/projects/([^/]*)/locations/([^/]*)/memberships/([^/]*)$"
-  scope_membership = { for val in setproduct(keys(var.namespace_ids), var.cluster_membership_ids) :
-  "${val[0]}-${val[1]}" => val }
+
+  scope_membership = { for idx, val in setproduct(keys(var.namespace_ids), var.cluster_membership_ids) :
+  "${val[0]}-${idx}" => val }
 }
 
 resource "random_string" "suffix" {

3-fleetscope/modules/env_baseline/mcg.tf

@@ -29,8 +29,8 @@ resource "google_gke_hub_feature" "mci" {
   }
 
   depends_on = [
-    google_gke_hub_feature.mcs,
-    google_gke_hub_feature.fleet-o11y
+    google_gke_hub_feature.mcs
+    # google_gke_hub_feature.fleet-o11y
   ]
 }
 
@@ -46,8 +46,8 @@ resource "google_project_service_identity" "fleet_mci_sa" {
   service  = "multiclusteringress.googleapis.com"
 
   depends_on = [
-    google_gke_hub_feature.mci,
-    google_gke_hub_feature.fleet-o11y
+    google_gke_hub_feature.mci
+    # google_gke_hub_feature.fleet-o11y
   ]
 }
 

3-fleetscope/modules/env_baseline/policy.tf

@@ -46,13 +46,13 @@ resource "google_gke_hub_feature" "poco_feature" {
 }
 
 resource "google_gke_hub_feature_membership" "poco_feature_member" {
-  for_each = toset(var.cluster_membership_ids)
+  for_each = local.cluster_membership_ids
   location = "global"
   project  = var.fleet_project_id
 
   feature             = google_gke_hub_feature.poco_feature.name
-  membership          = regex(local.membership_re, each.key)[2]
-  membership_location = regex(local.membership_re, each.key)[1]
+  membership          = regex(local.membership_re, each.value)[2]
+  membership_location = regex(local.membership_re, each.value)[1]
 
   policycontroller {
     policy_controller_hub_config {

3-fleetscope/modules/env_baseline/variables.tf

@@ -44,7 +44,6 @@ variable "cluster_membership_ids" {
   type        = list(string)
 }
 
-
 variable "additional_project_role_identities" {
   description = <<-EOF
   (Optional) A list of additional identities to assign roles at the project level for the fleet project. Use the following formats for specific Kubernetes identities:

5-appinfra/modules/cicd-pipeline/README.md

@@ -9,6 +9,7 @@
 | app\_build\_trigger\_yaml | Path to the Cloud Build YAML file for the application | `string` | n/a | yes |
 | buckets\_force\_destroy | When deleting the bucket for storing CICD artifacts, this boolean option will delete all contained objects. If false, Terraform will fail to delete buckets which contain objects. | `bool` | `false` | no |
 | ci\_build\_included\_files | (Optional) includedFiles are file glob matches using https://golang.org/pkg/path/filepath/#Match extended with support for **. If any of the files altered in the commit pass the ignoredFiles filter and includedFiles is empty, then as far as this filter is concerned, we should trigger the build. If any of the files altered in the commit pass the ignoredFiles filter and includedFiles is not empty, then we make sure that at least one of those files matches a includedFiles glob. If not, then we do not trigger a build. | `list(string)` | `[]` | no |
+| cluster\_services\_accounts | Cluster services accounts to be granted the Artifact Registry reader role. | `map(string)` | `{}` | no |
 | env\_cluster\_membership\_ids | Env Cluster Membership IDs | <pre>map(object({<br>    cluster_membership_ids = list(string)<br>  }))</pre> | n/a | yes |
 | project\_id | CI/CD project ID | `string` | n/a | yes |
 | region | CI/CD Region (e.g. us-central1) | `string` | n/a | yes |
@@ -19,7 +20,9 @@
 
 ## Outputs
 
-No outputs.
+| Name | Description |
+|------|-------------|
+| clouddeploy\_targets\_names | Cloud deploy targets names. |
 
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 

5-appinfra/modules/cicd-pipeline/artifact-registry.tf

@@ -26,10 +26,11 @@ resource "google_artifact_registry_repository" "container_registry" {
 }
 
 resource "google_artifact_registry_repository_iam_member" "member" {
-  for_each = {
-    "compute"      = "serviceAccount:${data.google_project.project.number}-compute@developer.gserviceaccount.com",
-    "cloud_deploy" = "serviceAccount:${google_service_account.cloud_deploy.email}",
-  }
+  for_each = merge({
+    cloud_deploy   = google_service_account.cloud_deploy.member,
+    cloud_build_si = google_project_service_identity.cloudbuild_service_identity.member,
+    compute        = data.google_compute_default_service_account.compute_service_identity.member,
+  }, var.cluster_services_accounts)
 
   project    = var.project_id
   location   = var.region

5-appinfra/modules/cicd-pipeline/cloud-deploy.tf

@@ -22,7 +22,7 @@ resource "google_clouddeploy_delivery_pipeline" "delivery-pipeline" {
       for_each = google_clouddeploy_target.clouddeploy_targets
       content {
         # TODO: use "production" profile once validated.
-        profiles  = [endswith(stages.value.name, "-development") ? "development" : (endswith(stages.value.name, "-nonproduction") ? "staging" : "production")]
+        profiles  = [endswith(stages.value.anthos_cluster[0].membership, "-development") ? "development" : (endswith(stages.value.name, "-nonproduction") ? "staging" : "production")]
         target_id = stages.value.name
       }
     }