diff --git a/1-bootstrap/main.tf b/1-bootstrap/main.tf index 1a2d8689..2897b072 100644 --- a/1-bootstrap/main.tf +++ b/1-bootstrap/main.tf @@ -40,8 +40,9 @@ locals { resource "google_sourcerepo_repository" "gcp_repo" { for_each = local.cb_config - project = var.project_id - name = each.value.repo_name + project = var.project_id + name = each.value.repo_name + create_ignore_already_exists = true } module "tfstate_bucket" { diff --git a/2-multitenant/modules/env_baseline/main.tf b/2-multitenant/modules/env_baseline/main.tf index db549193..5fb85013 100644 --- a/2-multitenant/modules/env_baseline/main.tf +++ b/2-multitenant/modules/env_baseline/main.tf @@ -21,11 +21,19 @@ locals { cluster_project_id = data.google_project.eab_cluster_project.project_id available_cidr_ranges = var.master_ipv4_cidr_blocks + subnets = { for idx, v in var.cluster_subnetworks : idx => v } + subnets_to_cidr = { for idx, subnet_key in keys(data.google_compute_subnetwork.default) : subnet_key => local.available_cidr_ranges[idx] } } +resource "google_project_service_identity" "compute_sa" { + provider = google-beta + project = local.cluster_project_id + service = "compute.googleapis.com" +} + // Create cluster project module "eab_cluster_project" { source = "terraform-google-modules/project-factory/google" @@ -84,6 +92,7 @@ module "cloud_armor" { type = "CLOUD_ARMOR" layer_7_ddos_defense_enable = true layer_7_ddos_defense_rule_visibility = "STANDARD" + user_ip_request_headers = [] pre_configured_rules = { "sqli_sensitivity_level_1" = { @@ -105,10 +114,50 @@ module "cloud_armor" { // Retrieve the subnetworks data "google_compute_subnetwork" "default" { - for_each = { for value in var.cluster_subnetworks : regex(local.subnetworks_re, value)[0] => value } + for_each = local.subnets self_link = each.value } +resource "google_project_service_identity" "gke_identity_cluster_project" { + provider = google-beta + project = local.cluster_project_id + service = "gkehub.googleapis.com" + depends_on = [module.eab_cluster_project] +} + +resource "google_project_service_identity" "mcsd_cluster_project" { + provider = google-beta + project = local.cluster_project_id + service = "multiclusterservicediscovery.googleapis.com" + depends_on = [module.eab_cluster_project] +} + +resource "google_project_iam_member" "gke_service_agent" { + project = local.cluster_project_id + role = "roles/gkehub.serviceAgent" + member = google_project_service_identity.gke_identity_cluster_project.member + depends_on = [module.eab_cluster_project] +} + +resource "google_project_service_identity" "fleet_meshconfig_sa" { + provider = google-beta + project = local.cluster_project_id + service = "meshconfig.googleapis.com" +} + +resource "google_project_iam_member" "servicemesh_service_agent" { + project = local.cluster_project_id + role = "roles/meshconfig.serviceAgent" + member = google_project_service_identity.fleet_meshconfig_sa.member + depends_on = [module.eab_cluster_project, google_project_service_identity.fleet_meshconfig_sa] +} + +resource "google_project_iam_member" "multiclusterdiscovery_service_agent" { + project = local.cluster_project_id + role = "roles/multiclusterservicediscovery.serviceAgent" + member = google_project_service_identity.mcsd_cluster_project.member +} + module "gke-standard" { source = "terraform-google-modules/kubernetes-engine/google//modules/beta-private-cluster" version = "~> 34.0" @@ -121,7 +170,7 @@ module "gke-standard" { region = each.value.region network_project_id = regex(local.projects_re, each.value.id)[0] network = regex(local.networks_re, each.value.network)[0] - subnetwork = each.value.name + subnetwork = regex(local.subnetworks_re, local.subnets[each.key])[0] ip_range_pods = each.value.secondary_ip_range[0].range_name ip_range_services = each.value.secondary_ip_range[1].range_name release_channel = var.cluster_release_channel @@ -176,14 +225,21 @@ module "gke-standard" { ] depends_on = [ - module.eab_cluster_project + module.eab_cluster_project, + google_project_iam_member.gke_service_agent, + google_project_iam_member.servicemesh_service_agent, + google_project_iam_member.multiclusterdiscovery_service_agent, + google_project_service_identity.compute_sa ] // Private Cluster Configuration enable_private_nodes = true enable_private_endpoint = true + fleet_project_grant_service_agent = true + deletion_protection = false # set to true to prevent the module from deleting the cluster on destroy + } module "gke-autopilot" { @@ -219,12 +275,24 @@ module "gke-autopilot" { } depends_on = [ - module.eab_cluster_project + module.eab_cluster_project, + google_project_iam_member.gke_service_agent, + google_project_iam_member.servicemesh_service_agent, + google_project_iam_member.multiclusterdiscovery_service_agent, + google_project_service_identity.compute_sa ] // Private Cluster Configuration enable_private_nodes = true enable_private_endpoint = true + fleet_project_grant_service_agent = true + deletion_protection = false # set to true to prevent the module from deleting the cluster on destroy } + +resource "time_sleep" "wait_service_cleanup" { + depends_on = [module.gke-autopilot.name, module.gke-standard.name] + + destroy_duration = "300s" +} diff --git a/2-multitenant/modules/env_baseline/outputs.tf b/2-multitenant/modules/env_baseline/outputs.tf index 629b6076..45bc9ab8 100644 --- a/2-multitenant/modules/env_baseline/outputs.tf +++ b/2-multitenant/modules/env_baseline/outputs.tf @@ -31,6 +31,8 @@ output "cluster_membership_ids" { output "cluster_project_id" { description = "Cluster Project ID" value = data.google_project.eab_cluster_project.project_id + + depends_on = [module.gke-standard, module.gke-autopilot] } output "cluster_project_number" { @@ -77,8 +79,8 @@ output "cluster_type" { output "cluster_service_accounts" { description = "The default service accounts used for nodes, if not overridden in node_pools." - value = setunion( - [for value in merge(module.gke-standard, module.gke-autopilot) : value.service_account], - [for value in module.eab_cluster_project : "${value.project_number}-compute@developer.gserviceaccount.com"] + value = merge( + { for i, value in merge(module.gke-standard, module.gke-autopilot) : "cluster_${var.env}_${i}" => value.service_account }, + { for i, value in module.eab_cluster_project : "project_${var.env}_${i}" => "${value.project_number}-compute@developer.gserviceaccount.com" } ) } diff --git a/2-multitenant/modules/env_baseline/versions.tf b/2-multitenant/modules/env_baseline/versions.tf index d3eef1c3..12e865bc 100644 --- a/2-multitenant/modules/env_baseline/versions.tf +++ b/2-multitenant/modules/env_baseline/versions.tf @@ -26,6 +26,10 @@ terraform { source = "hashicorp/google-beta" version = ">= 6.6, < 7" } + time = { + source = "hashicorp/time" + version = ">= 0.12.0" + } } provider_meta "google" { diff --git a/3-fleetscope/envs/development/main.tf b/3-fleetscope/envs/development/main.tf index 02d737ab..5a734577 100644 --- a/3-fleetscope/envs/development/main.tf +++ b/3-fleetscope/envs/development/main.tf @@ -18,10 +18,10 @@ locals { env = "development" } -import { - id = "projects/${local.cluster_project_id}/locations/global/features/fleetobservability" - to = module.env.google_gke_hub_feature.fleet-o11y -} +# import { +# id = "projects/${local.cluster_project_id}/locations/global/features/fleetobservability" +# to = module.env.google_gke_hub_feature.fleet-o11y +# } module "env" { source = "../../modules/env_baseline" diff --git a/3-fleetscope/envs/nonproduction/main.tf b/3-fleetscope/envs/nonproduction/main.tf index 87259e7a..6a9185ac 100644 --- a/3-fleetscope/envs/nonproduction/main.tf +++ b/3-fleetscope/envs/nonproduction/main.tf @@ -18,10 +18,10 @@ locals { env = "nonproduction" } -import { - id = "projects/${local.cluster_project_id}/locations/global/features/fleetobservability" - to = module.env.google_gke_hub_feature.fleet-o11y -} +# import { +# id = "projects/${local.cluster_project_id}/locations/global/features/fleetobservability" +# to = module.env.google_gke_hub_feature.fleet-o11y +# } module "env" { source = "../../modules/env_baseline" diff --git a/3-fleetscope/envs/production/main.tf b/3-fleetscope/envs/production/main.tf index c06e9bb3..994d4d3c 100644 --- a/3-fleetscope/envs/production/main.tf +++ b/3-fleetscope/envs/production/main.tf @@ -18,10 +18,10 @@ locals { env = "production" } -import { - id = "projects/${local.cluster_project_id}/locations/global/features/fleetobservability" - to = module.env.google_gke_hub_feature.fleet-o11y -} +# import { +# id = "projects/${local.cluster_project_id}/locations/global/features/fleetobservability" +# to = module.env.google_gke_hub_feature.fleet-o11y +# } module "env" { source = "../../modules/env_baseline" diff --git a/3-fleetscope/modules/env_baseline/acm.tf b/3-fleetscope/modules/env_baseline/acm.tf index 50b9add7..b2b7bc99 100644 --- a/3-fleetscope/modules/env_baseline/acm.tf +++ b/3-fleetscope/modules/env_baseline/acm.tf @@ -14,13 +14,18 @@ * limitations under the License. */ +locals { + cluster_membership_ids = { for k, v in var.cluster_membership_ids : k => v } +} + data "google_project" "cluster_project" { project_id = var.cluster_project_id } resource "google_sourcerepo_repository" "acm_repo" { - project = var.cluster_project_id - name = "eab-acm" + project = var.cluster_project_id + name = "eab-acm" + create_ignore_already_exists = true } resource "google_service_account" "root_reconciler" { @@ -36,13 +41,11 @@ resource "google_project_iam_member" "root_reconciler" { member = "serviceAccount:${google_service_account.root_reconciler.email}" } -resource "google_service_account_iam_binding" "workload_identity" { +resource "google_service_account_iam_member" "workload_identity" { service_account_id = google_service_account.root_reconciler.name - role = "roles/iam.workloadIdentityUser" - members = [ - "serviceAccount:${var.cluster_project_id}.svc.id.goog[config-management-system/root-reconciler]", - ] + role = "roles/iam.workloadIdentityUser" + member = "serviceAccount:${var.cluster_project_id}.svc.id.goog[config-management-system/root-reconciler]" } resource "google_gke_hub_feature" "acm_feature" { @@ -52,14 +55,14 @@ resource "google_gke_hub_feature" "acm_feature" { } resource "google_gke_hub_feature_membership" "acm_feature_member" { - for_each = toset(var.cluster_membership_ids) + for_each = local.cluster_membership_ids project = var.cluster_project_id location = "global" feature = google_gke_hub_feature.acm_feature.name - membership = regex(local.membership_re, each.key)[2] - membership_location = regex(local.membership_re, each.key)[1] + membership = regex(local.membership_re, each.value)[2] + membership_location = regex(local.membership_re, each.value)[1] configmanagement { version = "1.19.0" diff --git a/3-fleetscope/modules/env_baseline/asm.tf b/3-fleetscope/modules/env_baseline/asm.tf index 38042f0b..1d7e4435 100644 --- a/3-fleetscope/modules/env_baseline/asm.tf +++ b/3-fleetscope/modules/env_baseline/asm.tf @@ -33,11 +33,11 @@ resource "google_gke_hub_feature_membership" "mesh_feature_member" { project = var.fleet_project_id location = "global" - for_each = toset(var.cluster_membership_ids) + for_each = local.cluster_membership_ids feature = google_gke_hub_feature.mesh_feature.name - membership = regex(local.membership_re, each.key)[2] - membership_location = regex(local.membership_re, each.key)[1] + membership = regex(local.membership_re, each.value)[2] + membership_location = regex(local.membership_re, each.value)[1] mesh { management = "MANAGEMENT_AUTOMATIC" diff --git a/3-fleetscope/modules/env_baseline/log.tf b/3-fleetscope/modules/env_baseline/log.tf index 9be64ae9..8f707559 100644 --- a/3-fleetscope/modules/env_baseline/log.tf +++ b/3-fleetscope/modules/env_baseline/log.tf @@ -14,28 +14,28 @@ * limitations under the License. */ -resource "google_gke_hub_feature" "fleet-o11y" { - name = "fleetobservability" - project = var.fleet_project_id - location = "global" - spec { - fleetobservability { - logging_config { - default_config { - mode = "COPY" - } - fleet_scope_logs_config { - mode = "MOVE" - } - } - } - } +# resource "google_gke_hub_feature" "fleet-o11y" { +# name = "fleetobservability" +# project = var.fleet_project_id +# location = "global" +# spec { +# fleetobservability { +# logging_config { +# default_config { +# mode = "COPY" +# } +# fleet_scope_logs_config { +# mode = "MOVE" +# } +# } +# } +# } - depends_on = [ - google_gke_hub_feature.mesh_feature, - google_project_iam_member.fleet_logging_viewaccessor - ] -} +# depends_on = [ +# google_gke_hub_feature.mesh_feature, +# google_project_iam_member.fleet_logging_viewaccessor +# ] +# } resource "google_project_iam_member" "fleet_logging_viewaccessor" { for_each = var.namespace_ids diff --git a/3-fleetscope/modules/env_baseline/main.tf b/3-fleetscope/modules/env_baseline/main.tf index cb294b72..c1b73a5a 100644 --- a/3-fleetscope/modules/env_baseline/main.tf +++ b/3-fleetscope/modules/env_baseline/main.tf @@ -16,8 +16,9 @@ locals { membership_re = "//gkehub.googleapis.com/projects/([^/]*)/locations/([^/]*)/memberships/([^/]*)$" - scope_membership = { for val in setproduct(keys(var.namespace_ids), var.cluster_membership_ids) : - "${val[0]}-${val[1]}" => val } + + scope_membership = { for idx, val in setproduct(keys(var.namespace_ids), var.cluster_membership_ids) : + "${val[0]}-${idx}" => val } } resource "random_string" "suffix" { diff --git a/3-fleetscope/modules/env_baseline/mcg.tf b/3-fleetscope/modules/env_baseline/mcg.tf index aab95434..0c768f66 100644 --- a/3-fleetscope/modules/env_baseline/mcg.tf +++ b/3-fleetscope/modules/env_baseline/mcg.tf @@ -29,8 +29,8 @@ resource "google_gke_hub_feature" "mci" { } depends_on = [ - google_gke_hub_feature.mcs, - google_gke_hub_feature.fleet-o11y + google_gke_hub_feature.mcs + # google_gke_hub_feature.fleet-o11y ] } @@ -46,8 +46,8 @@ resource "google_project_service_identity" "fleet_mci_sa" { service = "multiclusteringress.googleapis.com" depends_on = [ - google_gke_hub_feature.mci, - google_gke_hub_feature.fleet-o11y + google_gke_hub_feature.mci + # google_gke_hub_feature.fleet-o11y ] } diff --git a/3-fleetscope/modules/env_baseline/policy.tf b/3-fleetscope/modules/env_baseline/policy.tf index 282fd69a..7420f899 100644 --- a/3-fleetscope/modules/env_baseline/policy.tf +++ b/3-fleetscope/modules/env_baseline/policy.tf @@ -46,13 +46,13 @@ resource "google_gke_hub_feature" "poco_feature" { } resource "google_gke_hub_feature_membership" "poco_feature_member" { - for_each = toset(var.cluster_membership_ids) + for_each = local.cluster_membership_ids location = "global" project = var.fleet_project_id feature = google_gke_hub_feature.poco_feature.name - membership = regex(local.membership_re, each.key)[2] - membership_location = regex(local.membership_re, each.key)[1] + membership = regex(local.membership_re, each.value)[2] + membership_location = regex(local.membership_re, each.value)[1] policycontroller { policy_controller_hub_config { diff --git a/3-fleetscope/modules/env_baseline/variables.tf b/3-fleetscope/modules/env_baseline/variables.tf index 96d419f9..aec99202 100644 --- a/3-fleetscope/modules/env_baseline/variables.tf +++ b/3-fleetscope/modules/env_baseline/variables.tf @@ -44,7 +44,6 @@ variable "cluster_membership_ids" { type = list(string) } - variable "additional_project_role_identities" { description = <<-EOF (Optional) A list of additional identities to assign roles at the project level for the fleet project. Use the following formats for specific Kubernetes identities: diff --git a/4-appfactory/envs/shared/iam.tf b/4-appfactory/envs/shared/iam.tf deleted file mode 100644 index 73098aad..00000000 --- a/4-appfactory/envs/shared/iam.tf +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - all_environments_cluster_service_accounts_iam_members = [for sa in local.cluster_service_accounts : "serviceAccount:${sa}"] - - expanded_cluster_service_accounts = flatten([ - for key in local.application_names : [ - for sa in local.all_environments_cluster_service_accounts_iam_members : { - app_name = key - cluster_sa_member = sa - } - ] - ]) -} - -// Assign artifactregistry reader to cluster service accounts -// This allows docker images on application projects to be downloaded on the cluster -resource "google_folder_iam_member" "admin" { - // for each app folder, create permissions for dev/non prod and prod cluster service accounts - for_each = tomap({ - for app_name_sa in local.expanded_cluster_service_accounts : "${app_name_sa.app_name}.${app_name_sa.cluster_sa_member}" => app_name_sa - }) - - folder = google_folder.app_folder[each.value.app_name].name - role = "roles/artifactregistry.reader" - member = each.value.cluster_sa_member -} diff --git a/4-appfactory/envs/shared/remote.tf b/4-appfactory/envs/shared/remote.tf index 06c78c90..e3cff20b 100644 --- a/4-appfactory/envs/shared/remote.tf +++ b/4-appfactory/envs/shared/remote.tf @@ -17,12 +17,11 @@ // These values are retrieved from the saved terraform state of the execution // of previous step using the terraform_remote_state data source. locals { - cluster_service_accounts = flatten([for state in data.terraform_remote_state.multitenant : state.outputs.cluster_service_accounts]) - cluster_projects_ids = [for state in data.terraform_remote_state.multitenant : state.outputs.cluster_project_id] - acronym = flatten([for state in data.terraform_remote_state.multitenant : state.outputs.acronyms])[0] - gar_project_id = data.terraform_remote_state.bootstrap.outputs.tf_project_id - gar_image_name = data.terraform_remote_state.bootstrap.outputs.tf_repository_name - gar_tag_version = data.terraform_remote_state.bootstrap.outputs.tf_tag_version_terraform + cluster_projects_ids = [for state in data.terraform_remote_state.multitenant : state.outputs.cluster_project_id] + acronym = flatten([for state in data.terraform_remote_state.multitenant : state.outputs.acronyms])[0] + gar_project_id = data.terraform_remote_state.bootstrap.outputs.tf_project_id + gar_image_name = data.terraform_remote_state.bootstrap.outputs.tf_repository_name + gar_tag_version = data.terraform_remote_state.bootstrap.outputs.tf_tag_version_terraform } data "terraform_remote_state" "multitenant" { diff --git a/4-appfactory/modules/app-group-baseline/main.tf b/4-appfactory/modules/app-group-baseline/main.tf index a9e97ce7..e1e10f1f 100644 --- a/4-appfactory/modules/app-group-baseline/main.tf +++ b/4-appfactory/modules/app-group-baseline/main.tf @@ -67,11 +67,35 @@ module "app_admin_project" { "sourcerepo.googleapis.com", "clouddeploy.googleapis.com" ] + + activate_api_identities = [ + { + api = "compute.googleapis.com", + roles = [] + }, + { + api = "cloudbuild.googleapis.com", + roles = [ + "roles/cloudbuild.builds.builder", + "roles/cloudbuild.connectionAdmin", + ] + }, + { + api = "workflows.googleapis.com", + roles = ["roles/workflows.serviceAgent"] + }, + { + api = "config.googleapis.com", + roles = ["roles/cloudconfig.serviceAgent"] + } + ] + } resource "google_sourcerepo_repository" "app_infra_repo" { - project = local.admin_project_id - name = "${var.service_name}-i-r" + project = local.admin_project_id + name = "${var.service_name}-i-r" + create_ignore_already_exists = true } module "tf_cloudbuild_workspace" { diff --git a/5-appinfra/apps/default-example/hello-world/envs/shared/main.tf b/5-appinfra/apps/default-example/hello-world/envs/shared/main.tf index 7bdbec84..36c79aaf 100644 --- a/5-appinfra/apps/default-example/hello-world/envs/shared/main.tf +++ b/5-appinfra/apps/default-example/hello-world/envs/shared/main.tf @@ -28,6 +28,7 @@ module "app" { project_id = local.app_admin_project region = var.region env_cluster_membership_ids = local.cluster_membership_ids + cluster_service_accounts = { for i, sa in local.cluster_service_accounts : (i) => "serviceAccount:${sa}" } service_name = local.service_name team_name = local.team_name diff --git a/5-appinfra/apps/default-example/hello-world/envs/shared/remote.tf b/5-appinfra/apps/default-example/hello-world/envs/shared/remote.tf index 2b6eb980..e349aabb 100644 --- a/5-appinfra/apps/default-example/hello-world/envs/shared/remote.tf +++ b/5-appinfra/apps/default-example/hello-world/envs/shared/remote.tf @@ -16,7 +16,15 @@ locals { cluster_membership_ids = { for state in data.terraform_remote_state.multitenant : (state.outputs.env) => { "cluster_membership_ids" = (state.outputs.cluster_membership_ids) } } - app_admin_project = data.terraform_remote_state.appfactory.outputs.app-group["default-example.hello-world"].app_admin_project_id + cluster_service_accounts = zipmap( + flatten( + [for item in data.terraform_remote_state.multitenant : keys(item.outputs.cluster_service_accounts)] + ), + flatten( + [for item in data.terraform_remote_state.multitenant : values(item.outputs.cluster_service_accounts)] + ) + ) + app_admin_project = data.terraform_remote_state.appfactory.outputs.app-group["default-example.hello-world"].app_admin_project_id } data "terraform_remote_state" "multitenant" { diff --git a/5-appinfra/modules/cicd-pipeline/README.md b/5-appinfra/modules/cicd-pipeline/README.md index 5ecbc60d..badd5a4d 100644 --- a/5-appinfra/modules/cicd-pipeline/README.md +++ b/5-appinfra/modules/cicd-pipeline/README.md @@ -9,6 +9,7 @@ | app\_build\_trigger\_yaml | Path to the Cloud Build YAML file for the application | `string` | n/a | yes | | buckets\_force\_destroy | When deleting the bucket for storing CICD artifacts, this boolean option will delete all contained objects. If false, Terraform will fail to delete buckets which contain objects. | `bool` | `false` | no | | ci\_build\_included\_files | (Optional) includedFiles are file glob matches using https://golang.org/pkg/path/filepath/#Match extended with support for **. If any of the files altered in the commit pass the ignoredFiles filter and includedFiles is empty, then as far as this filter is concerned, we should trigger the build. If any of the files altered in the commit pass the ignoredFiles filter and includedFiles is not empty, then we make sure that at least one of those files matches a includedFiles glob. If not, then we do not trigger a build. | `list(string)` | `[]` | no | +| cluster\_service\_accounts | Cluster services accounts to be granted the Artifact Registry reader role. | `map(string)` | n/a | yes | | env\_cluster\_membership\_ids | Env Cluster Membership IDs |
map(object({| n/a | yes | | project\_id | CI/CD project ID | `string` | n/a | yes | | region | CI/CD Region (e.g. us-central1) | `string` | n/a | yes | @@ -21,6 +22,7 @@ | Name | Description | |------|-------------| +| clouddeploy\_targets\_names | Cloud deploy targets names. | | service\_repository\_name | The Source Repository name. | | service\_repository\_project\_id | The Source Repository project id. | diff --git a/5-appinfra/modules/cicd-pipeline/artifact-registry.tf b/5-appinfra/modules/cicd-pipeline/artifact-registry.tf index d5daca5a..6d2a3648 100644 --- a/5-appinfra/modules/cicd-pipeline/artifact-registry.tf +++ b/5-appinfra/modules/cicd-pipeline/artifact-registry.tf @@ -26,10 +26,11 @@ resource "google_artifact_registry_repository" "container_registry" { } resource "google_artifact_registry_repository_iam_member" "member" { - for_each = { - "compute" = "serviceAccount:${data.google_project.project.number}-compute@developer.gserviceaccount.com", - "cloud_deploy" = "serviceAccount:${google_service_account.cloud_deploy.email}", - } + for_each = merge({ + cloud_deploy = google_service_account.cloud_deploy.member, + cloud_build_si = google_project_service_identity.cloudbuild_service_identity.member, + compute = data.google_compute_default_service_account.compute_service_identity.member, + }, var.cluster_service_accounts) project = var.project_id location = var.region diff --git a/5-appinfra/modules/cicd-pipeline/cloud-deploy.tf b/5-appinfra/modules/cicd-pipeline/cloud-deploy.tf index dbf215f3..e8e3643c 100644 --- a/5-appinfra/modules/cicd-pipeline/cloud-deploy.tf +++ b/5-appinfra/modules/cicd-pipeline/cloud-deploy.tf @@ -22,7 +22,7 @@ resource "google_clouddeploy_delivery_pipeline" "delivery-pipeline" { for_each = google_clouddeploy_target.clouddeploy_targets content { # TODO: use "production" profile once validated. - profiles = [endswith(stages.value.name, "-development") ? "development" : (endswith(stages.value.name, "-nonproduction") ? "staging" : "production")] + profiles = [endswith(stages.value.anthos_cluster[0].membership, "-development") ? "development" : (endswith(stages.value.name, "-nonproduction") ? "staging" : "production")] target_id = stages.value.name } } diff --git a/5-appinfra/modules/cicd-pipeline/main.tf b/5-appinfra/modules/cicd-pipeline/main.tf index f8f72f9c..7240d313 100644 --- a/5-appinfra/modules/cicd-pipeline/main.tf +++ b/5-appinfra/modules/cicd-pipeline/main.tf @@ -17,7 +17,27 @@ data "google_project" "project" { project_id = var.project_id } +resource "google_project_service_identity" "cloudbuild_service_identity" { + provider = google-beta + + project = var.project_id + service = "cloudbuild.googleapis.com" +} + +data "google_compute_default_service_account" "compute_service_identity" { + project = var.project_id +} + resource "google_sourcerepo_repository" "app_repo" { project = var.project_id name = var.repo_name + + create_ignore_already_exists = true +} + +resource "google_sourcerepo_repository_iam_member" "member" { + project = var.project_id + repository = google_sourcerepo_repository.app_repo.name + role = "roles/source.admin" + member = google_project_service_identity.cloudbuild_service_identity.member } diff --git a/5-appinfra/modules/cicd-pipeline/outputs.tf b/5-appinfra/modules/cicd-pipeline/outputs.tf index dd6818a9..fec70fd8 100644 --- a/5-appinfra/modules/cicd-pipeline/outputs.tf +++ b/5-appinfra/modules/cicd-pipeline/outputs.tf @@ -1,18 +1,21 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "clouddeploy_targets_names" { + description = "Cloud deploy targets names." + value = [for target in google_clouddeploy_target.clouddeploy_targets : target.name] +} output "service_repository_name" { description = "The Source Repository name." diff --git a/5-appinfra/modules/cicd-pipeline/pipelines.tf b/5-appinfra/modules/cicd-pipeline/pipelines.tf index 175334b2..47ea265f 100644 --- a/5-appinfra/modules/cicd-pipeline/pipelines.tf +++ b/5-appinfra/modules/cicd-pipeline/pipelines.tf @@ -17,6 +17,7 @@ locals { fleet_membership_re = "//gkehub.googleapis.com/(.*)$" + service_name_short = substr(var.service_name, 0, 4) } # cloud deploy service account @@ -28,14 +29,10 @@ resource "google_service_account" "cloud_deploy" { resource "google_clouddeploy_target" "clouddeploy_targets" { # one CloudDeploy target per cluster_membership_id defined in vars - for_each = merge([ - for key, value in var.env_cluster_membership_ids : { - for item in value.cluster_membership_ids : "${key}/${item}" => item - } - ]...) + for_each = local.memberships_map project = var.project_id - name = trimprefix(regex(local.membership_re, each.value)[2], "cluster-") + name = trimsuffix(substr("${local.service_name_short}-${trimprefix(regex(local.membership_re, each.value)[2], "cluster-")}", 0, 21), "-") location = var.region anthos_cluster { @@ -43,7 +40,7 @@ resource "google_clouddeploy_target" "clouddeploy_targets" { } execution_configs { - artifact_storage = "gs://${google_storage_bucket.delivery_artifacts[split("/", each.key)[0]].name}" + artifact_storage = "gs://${google_storage_bucket.delivery_artifacts[split("-", each.value)[length(split("-", each.value)) - 1]].name}" service_account = google_service_account.cloud_deploy.email usages = [ "RENDER", diff --git a/5-appinfra/modules/cicd-pipeline/project-iam-bindings.tf b/5-appinfra/modules/cicd-pipeline/project-iam-bindings.tf index f4b06141..ee4c6c27 100644 --- a/5-appinfra/modules/cicd-pipeline/project-iam-bindings.tf +++ b/5-appinfra/modules/cicd-pipeline/project-iam-bindings.tf @@ -13,67 +13,107 @@ # limitations under the License. locals { - cloud_build_sas = ["serviceAccount:${google_service_account.cloud_build.email}"] # cloud build service accounts used for CI - membership_re = "projects/([^/]*)/locations/([^/]*)/memberships/([^/]*)$" - gke_projects = distinct(flatten([ - for _, value in var.env_cluster_membership_ids : [ - for item in value.cluster_membership_ids : regex(local.membership_re, item)[0] - ] - ])) + membership_re = "projects/([^/]*)/locations/([^/]*)/memberships/([^/]*)$" + envs = keys(var.env_cluster_membership_ids) + + memberships = flatten([for i in local.envs : var.env_cluster_membership_ids[i].cluster_membership_ids]) + memberships_map = { for i, item in local.memberships : (i) => item } + gke_projects = { for i, item in local.memberships : (i) => regex(local.membership_re, item)[0] } +} + +resource "google_project_iam_member" "cloud_trace_agent" { + project = var.project_id + role = "roles/cloudtrace.agent" + + member = data.google_compute_default_service_account.compute_service_identity.member +} + +resource "google_project_iam_member" "metric_writer" { + project = var.project_id + role = "roles/monitoring.metricWriter" + + member = data.google_compute_default_service_account.compute_service_identity.member +} + +resource "google_project_iam_member" "log_writer" { + for_each = { + "compute" = data.google_compute_default_service_account.compute_service_identity.member, + "cloud_deploy" = google_service_account.cloud_deploy.member, + "cloud_build" = google_service_account.cloud_build.member, + } + project = var.project_id + role = "roles/logging.logWriter" + + member = each.value +} + +resource "google_project_iam_member" "builder" { + for_each = { + "cloud_build_service" = google_service_account.cloud_deploy.member, + "cloud_build" = google_service_account.cloud_build.member, + } + project = var.project_id + role = "roles/cloudbuild.builds.builder" + + member = each.value } -# authoritative project-iam-bindings to increase reproducibility -module "project-iam-bindings" { - source = "terraform-google-modules/iam/google//modules/projects_iam" - version = "~> 8.0" - projects = [var.project_id] - mode = "authoritative" - - bindings = { - "roles/cloudtrace.agent" = [ - "serviceAccount:${data.google_project.project.number}-compute@developer.gserviceaccount.com" - ], - "roles/monitoring.metricWriter" = [ - "serviceAccount:${data.google_project.project.number}-compute@developer.gserviceaccount.com" - ], - "roles/logging.logWriter" = setunion( - [ - "serviceAccount:${data.google_project.project.number}-compute@developer.gserviceaccount.com", - "serviceAccount:${google_service_account.cloud_deploy.email}" - ], - local.cloud_build_sas - ), - "roles/cloudbuild.builds.builder" = setunion( - [ - "serviceAccount:${data.google_project.project.number}@cloudbuild.gserviceaccount.com", - ], - local.cloud_build_sas - ), - "roles/gkehub.gatewayEditor" = [ - "serviceAccount:${google_service_account.cloud_deploy.email}" - ], - "roles/gkehub.viewer" = setunion( - local.cloud_build_sas, - [ - "serviceAccount:${google_service_account.cloud_deploy.email}" - ], - ), - "roles/clouddeploy.releaser" = local.cloud_build_sas, - "roles/container.developer" = [ - "serviceAccount:${google_service_account.cloud_deploy.email}" - ], - "roles/container.admin" = [ - "serviceAccount:${google_service_account.cloud_deploy.email}" - ], +resource "google_project_iam_member" "gateway_editor" { + for_each = { + "cloud_deploy" = google_service_account.cloud_deploy.member, + "cloud_build" = google_service_account.cloud_build.member, } + project = var.project_id + role = "roles/gkehub.gatewayEditor" + + member = each.value +} + +resource "google_project_iam_member" "gke_viewer" { + for_each = { + "cloud_deploy" = google_service_account.cloud_deploy.member, + "cloud_build" = google_service_account.cloud_build.member, + } + project = var.project_id + role = "roles/gkehub.viewer" + + member = each.value +} + +resource "google_project_iam_member" "cloud_deploy_releaser" { + project = var.project_id + role = "roles/clouddeploy.releaser" + + member = google_service_account.cloud_build.member } +resource "google_project_iam_member" "container_developer" { + for_each = { + "cloud_deploy" = google_service_account.cloud_deploy.member, + "cloud_build" = google_service_account.cloud_build.member, + } + project = var.project_id + role = "roles/container.developer" + + member = each.value +} + +resource "google_project_iam_member" "container_admin" { + for_each = { + "cloud_deploy" = google_service_account.cloud_deploy.member, + "cloud_build" = google_service_account.cloud_build.member, + } + project = var.project_id + role = "roles/container.admin" + + member = each.value +} // added to avoid overwriten of roles for each app service deploy service account, since GKE projects are shared between services module "cb-gke-project-iam-bindings" { source = "terraform-google-modules/iam/google//modules/member_iam" version = "~> 8.0" - for_each = toset(local.gke_projects) + for_each = local.gke_projects project_id = each.value project_roles = ["roles/container.admin", "roles/container.developer", "roles/gkehub.viewer", "roles/gkehub.gatewayEditor"] @@ -84,7 +124,7 @@ module "cb-gke-project-iam-bindings" { module "deploy-gke-project-iam-bindings" { source = "terraform-google-modules/iam/google//modules/member_iam" version = "~> 8.0" - for_each = toset(local.gke_projects) + for_each = local.gke_projects project_id = each.value project_roles = ["roles/container.admin", "roles/container.developer", "roles/gkehub.viewer", "roles/gkehub.gatewayEditor"] diff --git a/5-appinfra/modules/cicd-pipeline/service-accounts.tf b/5-appinfra/modules/cicd-pipeline/service-accounts.tf index f1880715..8320299f 100644 --- a/5-appinfra/modules/cicd-pipeline/service-accounts.tf +++ b/5-appinfra/modules/cicd-pipeline/service-accounts.tf @@ -34,3 +34,9 @@ resource "google_service_account_iam_member" "cloud_build_impersonate_cloud_depl role = "roles/iam.serviceAccountUser" member = "serviceAccount:${google_service_account.cloud_build.email}" } + +resource "google_service_account_iam_member" "cloud_build_token_creator_cloud_deploy" { + service_account_id = google_service_account.cloud_deploy.id + role = "roles/iam.serviceAccountTokenCreator" + member = "serviceAccount:${google_service_account.cloud_build.email}" +} diff --git a/5-appinfra/modules/cicd-pipeline/variables.tf b/5-appinfra/modules/cicd-pipeline/variables.tf index df455ef6..aacebd11 100644 --- a/5-appinfra/modules/cicd-pipeline/variables.tf +++ b/5-appinfra/modules/cicd-pipeline/variables.tf @@ -22,6 +22,11 @@ variable "region" { description = "CI/CD Region (e.g. us-central1)" } +variable "cluster_service_accounts" { + description = "Cluster services accounts to be granted the Artifact Registry reader role." + type = map(string) +} + variable "env_cluster_membership_ids" { description = "Env Cluster Membership IDs" type = map(object({ diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 7bcb95f6..4d1ef067 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -144,7 +144,7 @@ steps: [ "/bin/bash", "-c", - "cft test run TestSourceCymbalBank --stage verify --verbose", + "sleep 60s && cft test run TestSourceCymbalBank --stage verify --verbose", ] waitFor: - appinfra-apply @@ -156,7 +156,7 @@ steps: [ "/bin/bash", "-c", - "cft test run TestSourceCymbalShop --stage verify --verbose", + "sleep 60s && cft test run TestSourceCymbalShop --stage verify --verbose", ] waitFor: - appinfra-apply @@ -165,7 +165,7 @@ steps: - id: cymbal-bank-e2e name: "gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS" args: - ["/bin/bash", "-c", "cft test run TestCymbalBankE2E --stage verify --verbose"] + ["/bin/bash", "-c", "sleep 60 && cft test run TestCymbalBankE2E --stage verify --verbose"] waitFor: - appsource-verify-cymbal-bank @@ -239,6 +239,65 @@ steps: waitFor: - multitenant-teardown +# standalone single project example + - id: single-project-init + name: "gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS" + args: + [ + "/bin/bash", + "-c", + "cft test run TestStandaloneSingleProjectExample --stage init --verbose", + ] + waitFor: + - prepare + - id: single-project-apply + name: "gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS" + args: + [ + "/bin/bash", + "-c", + "cft test run TestStandaloneSingleProjectExample --stage apply --verbose", + ] + waitFor: + - single-project-init + - id: single-project-verify + name: "gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS" + args: + [ + "/bin/bash", + "-c", + "sleep 60s && cft test run TestStandaloneSingleProjectExample --stage verify --verbose", + ] + waitFor: + - single-project-apply + - id: single-project-source-verify + name: "gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS" + args: + [ + "/bin/bash", + "-c", + "sleep 60s && cft test run TestSingleProjectSourceCymbalBank --stage verify --verbose", + ] + waitFor: + - single-project-apply + - single-project-verify + - id: app-single-project-e2e + name: "gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS" + args: + ["/bin/bash", "-c", "cft test run TestAppE2ECymbalBankSingleProject --stage verify --verbose"] + waitFor: + - single-project-source-verify + - id: single-project-teardown + name: "gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS" + args: + [ + "/bin/bash", + "-c", + "cft test run TestStandaloneSingleProjectExample --stage teardown --verbose", + ] + waitFor: + - app-single-project-e2e + tags: - "ci" - "integration" diff --git a/examples/cymbal-bank/5-appinfra/cymbal-bank/accounts-contacts/envs/shared/main.tf b/examples/cymbal-bank/5-appinfra/cymbal-bank/accounts-contacts/envs/shared/main.tf index 9127dc53..5b62f199 100644 --- a/examples/cymbal-bank/5-appinfra/cymbal-bank/accounts-contacts/envs/shared/main.tf +++ b/examples/cymbal-bank/5-appinfra/cymbal-bank/accounts-contacts/envs/shared/main.tf @@ -29,6 +29,8 @@ module "app" { region = var.region env_cluster_membership_ids = local.cluster_membership_ids + cluster_service_accounts = { for i, sa in local.cluster_service_accounts : (i) => "serviceAccount:${sa}" } + service_name = local.service_name team_name = local.team_name repo_name = local.repo_name diff --git a/examples/cymbal-bank/5-appinfra/cymbal-bank/accounts-contacts/envs/shared/outputs.tf b/examples/cymbal-bank/5-appinfra/cymbal-bank/accounts-contacts/envs/shared/outputs.tf index 425de4a6..0ccaed18 100644 --- a/examples/cymbal-bank/5-appinfra/cymbal-bank/accounts-contacts/envs/shared/outputs.tf +++ b/examples/cymbal-bank/5-appinfra/cymbal-bank/accounts-contacts/envs/shared/outputs.tf @@ -14,6 +14,11 @@ * limitations under the License. */ +output "clouddeploy_targets_names" { + description = "Cloud deploy targets names." + value = module.app.clouddeploy_targets_names +} + output "service_repository_name" { description = "The Source Repository name." value = module.app.service_repository_name diff --git a/examples/cymbal-bank/5-appinfra/cymbal-bank/accounts-contacts/envs/shared/remote.tf b/examples/cymbal-bank/5-appinfra/cymbal-bank/accounts-contacts/envs/shared/remote.tf index 42a3211a..583f7369 100644 --- a/examples/cymbal-bank/5-appinfra/cymbal-bank/accounts-contacts/envs/shared/remote.tf +++ b/examples/cymbal-bank/5-appinfra/cymbal-bank/accounts-contacts/envs/shared/remote.tf @@ -16,7 +16,15 @@ locals { cluster_membership_ids = { for state in data.terraform_remote_state.multitenant : (state.outputs.env) => { "cluster_membership_ids" = (state.outputs.cluster_membership_ids) } } - app_admin_project = data.terraform_remote_state.appfactory.outputs.app-group["cymbal-bank.contacts"].app_admin_project_id + cluster_service_accounts = zipmap( + flatten( + [for item in data.terraform_remote_state.multitenant : keys(item.outputs.cluster_service_accounts)] + ), + flatten( + [for item in data.terraform_remote_state.multitenant : values(item.outputs.cluster_service_accounts)] + ) + ) + app_admin_project = data.terraform_remote_state.appfactory.outputs.app-group["cymbal-bank.contacts"].app_admin_project_id } data "terraform_remote_state" "multitenant" { diff --git a/examples/cymbal-bank/5-appinfra/cymbal-bank/accounts-userservice/envs/shared/main.tf b/examples/cymbal-bank/5-appinfra/cymbal-bank/accounts-userservice/envs/shared/main.tf index b51ef30d..9b016eed 100644 --- a/examples/cymbal-bank/5-appinfra/cymbal-bank/accounts-userservice/envs/shared/main.tf +++ b/examples/cymbal-bank/5-appinfra/cymbal-bank/accounts-userservice/envs/shared/main.tf @@ -29,6 +29,8 @@ module "app" { region = var.region env_cluster_membership_ids = local.cluster_membership_ids + cluster_service_accounts = { for i, sa in local.cluster_service_accounts : (i) => "serviceAccount:${sa}" } + service_name = local.service_name team_name = local.team_name repo_name = local.repo_name diff --git a/examples/cymbal-bank/5-appinfra/cymbal-bank/accounts-userservice/envs/shared/outputs.tf b/examples/cymbal-bank/5-appinfra/cymbal-bank/accounts-userservice/envs/shared/outputs.tf index 425de4a6..0ccaed18 100644 --- a/examples/cymbal-bank/5-appinfra/cymbal-bank/accounts-userservice/envs/shared/outputs.tf +++ b/examples/cymbal-bank/5-appinfra/cymbal-bank/accounts-userservice/envs/shared/outputs.tf @@ -14,6 +14,11 @@ * limitations under the License. */ +output "clouddeploy_targets_names" { + description = "Cloud deploy targets names." + value = module.app.clouddeploy_targets_names +} + output "service_repository_name" { description = "The Source Repository name." value = module.app.service_repository_name diff --git a/examples/cymbal-bank/5-appinfra/cymbal-bank/accounts-userservice/envs/shared/remote.tf b/examples/cymbal-bank/5-appinfra/cymbal-bank/accounts-userservice/envs/shared/remote.tf index 3c13674f..6901bae1 100644 --- a/examples/cymbal-bank/5-appinfra/cymbal-bank/accounts-userservice/envs/shared/remote.tf +++ b/examples/cymbal-bank/5-appinfra/cymbal-bank/accounts-userservice/envs/shared/remote.tf @@ -16,7 +16,15 @@ locals { cluster_membership_ids = { for state in data.terraform_remote_state.multitenant : (state.outputs.env) => { "cluster_membership_ids" = (state.outputs.cluster_membership_ids) } } - app_admin_project = data.terraform_remote_state.appfactory.outputs.app-group["cymbal-bank.userservice"].app_admin_project_id + cluster_service_accounts = zipmap( + flatten( + [for item in data.terraform_remote_state.multitenant : keys(item.outputs.cluster_service_accounts)] + ), + flatten( + [for item in data.terraform_remote_state.multitenant : values(item.outputs.cluster_service_accounts)] + ) + ) + app_admin_project = data.terraform_remote_state.appfactory.outputs.app-group["cymbal-bank.userservice"].app_admin_project_id } data "terraform_remote_state" "multitenant" { diff --git a/examples/cymbal-bank/5-appinfra/cymbal-bank/frontend/envs/shared/main.tf b/examples/cymbal-bank/5-appinfra/cymbal-bank/frontend/envs/shared/main.tf index b29f6e6a..e067ba7b 100644 --- a/examples/cymbal-bank/5-appinfra/cymbal-bank/frontend/envs/shared/main.tf +++ b/examples/cymbal-bank/5-appinfra/cymbal-bank/frontend/envs/shared/main.tf @@ -29,6 +29,8 @@ module "app" { region = var.region env_cluster_membership_ids = local.cluster_membership_ids + cluster_service_accounts = { for i, sa in local.cluster_service_accounts : (i) => "serviceAccount:${sa}" } + service_name = local.service_name team_name = local.team_name repo_name = local.repo_name diff --git a/examples/cymbal-bank/5-appinfra/cymbal-bank/frontend/envs/shared/outputs.tf b/examples/cymbal-bank/5-appinfra/cymbal-bank/frontend/envs/shared/outputs.tf index 425de4a6..0ccaed18 100644 --- a/examples/cymbal-bank/5-appinfra/cymbal-bank/frontend/envs/shared/outputs.tf +++ b/examples/cymbal-bank/5-appinfra/cymbal-bank/frontend/envs/shared/outputs.tf @@ -14,6 +14,11 @@ * limitations under the License. */ +output "clouddeploy_targets_names" { + description = "Cloud deploy targets names." + value = module.app.clouddeploy_targets_names +} + output "service_repository_name" { description = "The Source Repository name." value = module.app.service_repository_name diff --git a/examples/cymbal-bank/5-appinfra/cymbal-bank/frontend/envs/shared/remote.tf b/examples/cymbal-bank/5-appinfra/cymbal-bank/frontend/envs/shared/remote.tf index 3cfc9755..72ca01da 100644 --- a/examples/cymbal-bank/5-appinfra/cymbal-bank/frontend/envs/shared/remote.tf +++ b/examples/cymbal-bank/5-appinfra/cymbal-bank/frontend/envs/shared/remote.tf @@ -16,7 +16,15 @@ locals { cluster_membership_ids = { for state in data.terraform_remote_state.multitenant : (state.outputs.env) => { "cluster_membership_ids" = (state.outputs.cluster_membership_ids) } } - app_admin_project = data.terraform_remote_state.appfactory.outputs.app-group["cymbal-bank.frontend"].app_admin_project_id + cluster_service_accounts = zipmap( + flatten( + [for item in data.terraform_remote_state.multitenant : keys(item.outputs.cluster_service_accounts)] + ), + flatten( + [for item in data.terraform_remote_state.multitenant : values(item.outputs.cluster_service_accounts)] + ) + ) + app_admin_project = data.terraform_remote_state.appfactory.outputs.app-group["cymbal-bank.frontend"].app_admin_project_id } data "terraform_remote_state" "multitenant" { diff --git a/examples/cymbal-bank/5-appinfra/cymbal-bank/ledger-balancereader/envs/shared/main.tf b/examples/cymbal-bank/5-appinfra/cymbal-bank/ledger-balancereader/envs/shared/main.tf index 40489283..d7c071b2 100644 --- a/examples/cymbal-bank/5-appinfra/cymbal-bank/ledger-balancereader/envs/shared/main.tf +++ b/examples/cymbal-bank/5-appinfra/cymbal-bank/ledger-balancereader/envs/shared/main.tf @@ -29,6 +29,8 @@ module "app" { region = var.region env_cluster_membership_ids = local.cluster_membership_ids + cluster_service_accounts = { for i, sa in local.cluster_service_accounts : (i) => "serviceAccount:${sa}" } + service_name = local.service_name team_name = local.team_name repo_name = local.repo_name diff --git a/examples/cymbal-bank/5-appinfra/cymbal-bank/ledger-balancereader/envs/shared/outputs.tf b/examples/cymbal-bank/5-appinfra/cymbal-bank/ledger-balancereader/envs/shared/outputs.tf index 425de4a6..0ccaed18 100644 --- a/examples/cymbal-bank/5-appinfra/cymbal-bank/ledger-balancereader/envs/shared/outputs.tf +++ b/examples/cymbal-bank/5-appinfra/cymbal-bank/ledger-balancereader/envs/shared/outputs.tf @@ -14,6 +14,11 @@ * limitations under the License. */ +output "clouddeploy_targets_names" { + description = "Cloud deploy targets names." + value = module.app.clouddeploy_targets_names +} + output "service_repository_name" { description = "The Source Repository name." value = module.app.service_repository_name diff --git a/examples/cymbal-bank/5-appinfra/cymbal-bank/ledger-balancereader/envs/shared/remote.tf b/examples/cymbal-bank/5-appinfra/cymbal-bank/ledger-balancereader/envs/shared/remote.tf index b059699a..f4db8a50 100644 --- a/examples/cymbal-bank/5-appinfra/cymbal-bank/ledger-balancereader/envs/shared/remote.tf +++ b/examples/cymbal-bank/5-appinfra/cymbal-bank/ledger-balancereader/envs/shared/remote.tf @@ -16,7 +16,15 @@ locals { cluster_membership_ids = { for state in data.terraform_remote_state.multitenant : (state.outputs.env) => { "cluster_membership_ids" = (state.outputs.cluster_membership_ids) } } - app_admin_project = data.terraform_remote_state.appfactory.outputs.app-group["cymbal-bank.balancereader"].app_admin_project_id + cluster_service_accounts = zipmap( + flatten( + [for item in data.terraform_remote_state.multitenant : keys(item.outputs.cluster_service_accounts)] + ), + flatten( + [for item in data.terraform_remote_state.multitenant : values(item.outputs.cluster_service_accounts)] + ) + ) + app_admin_project = data.terraform_remote_state.appfactory.outputs.app-group["cymbal-bank.balancereader"].app_admin_project_id } data "terraform_remote_state" "multitenant" { diff --git a/examples/cymbal-bank/5-appinfra/cymbal-bank/ledger-ledgerwriter/envs/shared/main.tf b/examples/cymbal-bank/5-appinfra/cymbal-bank/ledger-ledgerwriter/envs/shared/main.tf index a937e8ad..7d47f697 100644 --- a/examples/cymbal-bank/5-appinfra/cymbal-bank/ledger-ledgerwriter/envs/shared/main.tf +++ b/examples/cymbal-bank/5-appinfra/cymbal-bank/ledger-ledgerwriter/envs/shared/main.tf @@ -29,6 +29,8 @@ module "app" { region = var.region env_cluster_membership_ids = local.cluster_membership_ids + cluster_service_accounts = { for i, sa in local.cluster_service_accounts : (i) => "serviceAccount:${sa}" } + service_name = local.service_name team_name = local.team_name repo_name = local.repo_name diff --git a/examples/cymbal-bank/5-appinfra/cymbal-bank/ledger-ledgerwriter/envs/shared/outputs.tf b/examples/cymbal-bank/5-appinfra/cymbal-bank/ledger-ledgerwriter/envs/shared/outputs.tf index 425de4a6..0ccaed18 100644 --- a/examples/cymbal-bank/5-appinfra/cymbal-bank/ledger-ledgerwriter/envs/shared/outputs.tf +++ b/examples/cymbal-bank/5-appinfra/cymbal-bank/ledger-ledgerwriter/envs/shared/outputs.tf @@ -14,6 +14,11 @@ * limitations under the License. */ +output "clouddeploy_targets_names" { + description = "Cloud deploy targets names." + value = module.app.clouddeploy_targets_names +} + output "service_repository_name" { description = "The Source Repository name." value = module.app.service_repository_name diff --git a/examples/cymbal-bank/5-appinfra/cymbal-bank/ledger-ledgerwriter/envs/shared/remote.tf b/examples/cymbal-bank/5-appinfra/cymbal-bank/ledger-ledgerwriter/envs/shared/remote.tf index 76229fec..8c592958 100644 --- a/examples/cymbal-bank/5-appinfra/cymbal-bank/ledger-ledgerwriter/envs/shared/remote.tf +++ b/examples/cymbal-bank/5-appinfra/cymbal-bank/ledger-ledgerwriter/envs/shared/remote.tf @@ -16,7 +16,15 @@ locals { cluster_membership_ids = { for state in data.terraform_remote_state.multitenant : (state.outputs.env) => { "cluster_membership_ids" = (state.outputs.cluster_membership_ids) } } - app_admin_project = data.terraform_remote_state.appfactory.outputs.app-group["cymbal-bank.ledgerwriter"].app_admin_project_id + cluster_service_accounts = zipmap( + flatten( + [for item in data.terraform_remote_state.multitenant : keys(item.outputs.cluster_service_accounts)] + ), + flatten( + [for item in data.terraform_remote_state.multitenant : values(item.outputs.cluster_service_accounts)] + ) + ) + app_admin_project = data.terraform_remote_state.appfactory.outputs.app-group["cymbal-bank.ledgerwriter"].app_admin_project_id } data "terraform_remote_state" "multitenant" { diff --git a/examples/cymbal-bank/5-appinfra/cymbal-bank/ledger-transactionhistory/envs/shared/main.tf b/examples/cymbal-bank/5-appinfra/cymbal-bank/ledger-transactionhistory/envs/shared/main.tf index 3490c244..036338d8 100644 --- a/examples/cymbal-bank/5-appinfra/cymbal-bank/ledger-transactionhistory/envs/shared/main.tf +++ b/examples/cymbal-bank/5-appinfra/cymbal-bank/ledger-transactionhistory/envs/shared/main.tf @@ -29,6 +29,8 @@ module "app" { region = var.region env_cluster_membership_ids = local.cluster_membership_ids + cluster_service_accounts = { for i, sa in local.cluster_service_accounts : (i) => "serviceAccount:${sa}" } + service_name = local.service_name team_name = local.team_name repo_name = local.repo_name diff --git a/examples/cymbal-bank/5-appinfra/cymbal-bank/ledger-transactionhistory/envs/shared/outputs.tf b/examples/cymbal-bank/5-appinfra/cymbal-bank/ledger-transactionhistory/envs/shared/outputs.tf index 425de4a6..0ccaed18 100644 --- a/examples/cymbal-bank/5-appinfra/cymbal-bank/ledger-transactionhistory/envs/shared/outputs.tf +++ b/examples/cymbal-bank/5-appinfra/cymbal-bank/ledger-transactionhistory/envs/shared/outputs.tf @@ -14,6 +14,11 @@ * limitations under the License. */ +output "clouddeploy_targets_names" { + description = "Cloud deploy targets names." + value = module.app.clouddeploy_targets_names +} + output "service_repository_name" { description = "The Source Repository name." value = module.app.service_repository_name diff --git a/examples/cymbal-bank/5-appinfra/cymbal-bank/ledger-transactionhistory/envs/shared/remote.tf b/examples/cymbal-bank/5-appinfra/cymbal-bank/ledger-transactionhistory/envs/shared/remote.tf index 813bf1d0..10cc483f 100644 --- a/examples/cymbal-bank/5-appinfra/cymbal-bank/ledger-transactionhistory/envs/shared/remote.tf +++ b/examples/cymbal-bank/5-appinfra/cymbal-bank/ledger-transactionhistory/envs/shared/remote.tf @@ -16,7 +16,15 @@ locals { cluster_membership_ids = { for state in data.terraform_remote_state.multitenant : (state.outputs.env) => { "cluster_membership_ids" = (state.outputs.cluster_membership_ids) } } - app_admin_project = data.terraform_remote_state.appfactory.outputs.app-group["cymbal-bank.transactionhistory"].app_admin_project_id + cluster_service_accounts = zipmap( + flatten( + [for item in data.terraform_remote_state.multitenant : keys(item.outputs.cluster_service_accounts)] + ), + flatten( + [for item in data.terraform_remote_state.multitenant : values(item.outputs.cluster_service_accounts)] + ) + ) + app_admin_project = data.terraform_remote_state.appfactory.outputs.app-group["cymbal-bank.transactionhistory"].app_admin_project_id } data "terraform_remote_state" "multitenant" { diff --git a/examples/cymbal-shop/5-appinfra/cymbal-shop/cymbalshop/envs/shared/main.tf b/examples/cymbal-shop/5-appinfra/cymbal-shop/cymbalshop/envs/shared/main.tf index f74fbaf2..ec4203d3 100644 --- a/examples/cymbal-shop/5-appinfra/cymbal-shop/cymbalshop/envs/shared/main.tf +++ b/examples/cymbal-shop/5-appinfra/cymbal-shop/cymbalshop/envs/shared/main.tf @@ -29,6 +29,9 @@ module "app" { region = var.region env_cluster_membership_ids = local.cluster_membership_ids + cluster_service_accounts = { for i, sa in local.cluster_service_accounts : (i) => "serviceAccount:${sa}" } + + service_name = local.service_name team_name = local.team_name repo_name = local.repo_name diff --git a/examples/cymbal-shop/5-appinfra/cymbal-shop/cymbalshop/envs/shared/outputs.tf b/examples/cymbal-shop/5-appinfra/cymbal-shop/cymbalshop/envs/shared/outputs.tf index dd112913..0ccaed18 100644 --- a/examples/cymbal-shop/5-appinfra/cymbal-shop/cymbalshop/envs/shared/outputs.tf +++ b/examples/cymbal-shop/5-appinfra/cymbal-shop/cymbalshop/envs/shared/outputs.tf @@ -13,3 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +output "clouddeploy_targets_names" { + description = "Cloud deploy targets names." + value = module.app.clouddeploy_targets_names +} + +output "service_repository_name" { + description = "The Source Repository name." + value = module.app.service_repository_name +} + +output "service_repository_project_id" { + description = "The Source Repository project id." + value = module.app.service_repository_project_id +} diff --git a/examples/cymbal-shop/5-appinfra/cymbal-shop/cymbalshop/envs/shared/remote.tf b/examples/cymbal-shop/5-appinfra/cymbal-shop/cymbalshop/envs/shared/remote.tf index 6e1841d2..f3014e40 100644 --- a/examples/cymbal-shop/5-appinfra/cymbal-shop/cymbalshop/envs/shared/remote.tf +++ b/examples/cymbal-shop/5-appinfra/cymbal-shop/cymbalshop/envs/shared/remote.tf @@ -16,7 +16,15 @@ locals { cluster_membership_ids = { for state in data.terraform_remote_state.multitenant : (state.outputs.env) => { "cluster_membership_ids" = (state.outputs.cluster_membership_ids) } } - app_admin_project = data.terraform_remote_state.appfactory.outputs.app-group["cymbal-shop.cymbalshop"].app_admin_project_id + cluster_service_accounts = zipmap( + flatten( + [for item in data.terraform_remote_state.multitenant : keys(item.outputs.cluster_service_accounts)] + ), + flatten( + [for item in data.terraform_remote_state.multitenant : values(item.outputs.cluster_service_accounts)] + ) + ) + app_admin_project = data.terraform_remote_state.appfactory.outputs.app-group["cymbal-shop.cymbalshop"].app_admin_project_id } data "terraform_remote_state" "multitenant" { diff --git a/examples/standalone_single_project/0-setup.tf b/examples/standalone_single_project/0-setup.tf new file mode 100644 index 00000000..d7d11e80 --- /dev/null +++ b/examples/standalone_single_project/0-setup.tf @@ -0,0 +1,96 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# Setup + +module "vpc" { + source = "terraform-google-modules/network/google" + version = "~> 9.0" + + project_id = var.project_id + network_name = "eab-vpc-${local.env}" + shared_vpc_host = false + + egress_rules = [ + { + name = "allow-private-google-access" + priority = 200 + destination_ranges = [ + "34.126.0.0/18", + "199.36.153.8/30", + ] + allow = [ + { + protocol = "tcp" + ports = ["443"] + } + ] + }, + { + name = "allow-private-google-access-ipv6" + priority = 200 + destination_ranges = [ + "2600:2d00:0002:2000::/64", + "2001:4860:8040::/42" + ] + allow = [ + { + protocol = "tcp" + ports = ["443"] + } + ] + } + ] + + subnets = [ + { + subnet_name = "eab-${local.short_env}-region01" + subnet_ip = "10.10.10.0/24" + subnet_region = "us-central1" + subnet_private_access = true + }, + { + subnet_name = "eab-${local.short_env}-region02" + subnet_ip = "10.10.20.0/24" + subnet_region = "us-east4" + subnet_private_access = true + }, + ] + + secondary_ranges = { + "eab-${local.short_env}-region01" = [ + { + range_name = "eab-${local.short_env}-region01-secondary-01" + ip_cidr_range = "192.168.0.0/18" + }, + { + range_name = "eab-${local.short_env}-region01-secondary-02" + ip_cidr_range = "192.168.64.0/18" + }, + ] + + "eab-${local.short_env}-region02" = [ + { + range_name = "eab-${local.short_env}-region02-secondary-01" + ip_cidr_range = "192.168.128.0/18" + }, + { + range_name = "eab-${local.short_env}-region02-secondary-02" + ip_cidr_range = "192.168.192.0/18" + }, + ] + } +} diff --git a/examples/standalone_single_project/2-multitenant.tf b/examples/standalone_single_project/2-multitenant.tf new file mode 100644 index 00000000..613a38a0 --- /dev/null +++ b/examples/standalone_single_project/2-multitenant.tf @@ -0,0 +1,48 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# 2-multitenantl + +locals { + env = "development" + short_env = "develop" + + apps = { + "cymbal-bank" : { + "ip_address_names" : [ + "frontend-ip", + ] + "certificates" : { + "frontend-example-com" : ["frontend.example.com"] + } + "acronym" = "cb", + } + } +} + +module "multitenant_infra" { + source = "../../2-multitenant/modules/env_baseline" + + apps = local.apps + cluster_subnetworks = module.vpc.subnets_self_links + network_project_id = var.project_id + env = local.env + create_cluster_project = false + # ignore below vars because we are reusing an existing project + org_id = null + folder_id = null + billing_account = null +} diff --git a/examples/standalone_single_project/3-fleetscope.tf b/examples/standalone_single_project/3-fleetscope.tf new file mode 100644 index 00000000..481173d3 --- /dev/null +++ b/examples/standalone_single_project/3-fleetscope.tf @@ -0,0 +1,33 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# 3-fleetscope +locals { + fleet_project_id = module.multitenant_infra.fleet_project_id + cluster_project_id = module.multitenant_infra.cluster_project_id + network_project_id = module.multitenant_infra.network_project_id +} + +module "fleetscope_infra" { + source = "../../3-fleetscope/modules/env_baseline" + + env = local.env + cluster_project_id = local.cluster_project_id + network_project_id = local.network_project_id + fleet_project_id = local.fleet_project_id + namespace_ids = var.teams + cluster_membership_ids = module.multitenant_infra.cluster_membership_ids +} diff --git a/examples/standalone_single_project/5-appinfra.tf b/examples/standalone_single_project/5-appinfra.tf new file mode 100644 index 00000000..62f4142a --- /dev/null +++ b/examples/standalone_single_project/5-appinfra.tf @@ -0,0 +1,85 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# 5-appinfra + +# app_01 +locals { + + cluster_membership_ids = { (local.env) : { "cluster_membership_ids" : module.multitenant_infra.cluster_membership_ids } } + cicd_apps = { "contacts" = { + application_name = "cymbal-bank" + service_name = "contacts" + team_name = "accounts" + repo_branch = "main" + }, + "userservice" = { + application_name = "cymbal-bank" + service_name = "userservice" + team_name = "accounts" + repo_branch = "main" + }, + "frontend" = { + application_name = "cymbal-bank" + service_name = "frontend" + team_name = "frontend" + repo_branch = "main" + }, + "balancereader" = { + application_name = "cymbal-bank" + service_name = "balancereader" + team_name = "ledger" + repo_branch = "main" + }, + "ledgerwriter" = { + application_name = "cymbal-bank" + service_name = "ledgerwriter" + team_name = "ledger" + repo_branch = "main" + }, + "transactionhistory" = { + application_name = "cymbal-bank" + service_name = "transactionhistory" + team_name = "ledger" + repo_branch = "main" + }, + } +} + +module "cicd" { + source = "../../5-appinfra/modules/cicd-pipeline" + for_each = local.cicd_apps + + project_id = var.project_id + region = var.region + env_cluster_membership_ids = local.cluster_membership_ids + cluster_service_accounts = { for i, sa in module.multitenant_infra.cluster_service_accounts : (i) => "serviceAccount:${sa}" } + + service_name = each.value.service_name + team_name = each.value.team_name + repo_name = each.value.team_name != each.value.service_name ? "eab-${each.value.application_name}-${each.value.team_name}-${each.value.service_name}" : "eab-${each.value.application_name}-${each.value.service_name}" + repo_branch = each.value.repo_branch + app_build_trigger_yaml = "src/${each.value.team_name}/cloudbuild.yaml" + + additional_substitutions = { + _SERVICE = each.value.service_name + _TEAM = each.value.team_name + } + + ci_build_included_files = ["src/${each.value.team_name}/**", "src/components/**"] + + buckets_force_destroy = true +} diff --git a/examples/standalone_single_project/README.md b/examples/standalone_single_project/README.md new file mode 100644 index 00000000..8ea8a221 --- /dev/null +++ b/examples/standalone_single_project/README.md @@ -0,0 +1,32 @@ +# Standalone Single-Project Example +The standalone example deploys the entire enterprise application blueprint into a single project for the purposes of simplified demonstation. Do not use this example for production deployments, as it lacks robust separation of duties and least-privileged permissions present in the standard multi-stage deployment. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| project\_id | Google Cloud project ID in which to deploy all example resources | `string` | n/a | yes | +| region | Google Cloud region for deployments | `string` | `"us-central1"` | no | +| teams | A map of string at the format {"namespace" = "groupEmail"} | `map(string)` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| acronyms | App Acronyms | +| app\_certificates | App Certificates | +| app\_infos | App infos (name, services, team). | +| app\_ip\_addresses | App IP Addresses | +| clouddeploy\_targets\_names | Cloud deploy targets names. | +| cluster\_membership\_ids | GKE cluster membership IDs | +| cluster\_project\_id | Cluster Project ID | +| cluster\_project\_number | Cluster Project ID | +| cluster\_regions | Regions with clusters | +| cluster\_service\_accounts | The default service accounts used for nodes, if not overridden in node\_pools. | +| cluster\_type | Cluster type | +| env | Environment | +| fleet\_project\_id | Fleet Project ID | +| network\_project\_id | Network Project ID | + + diff --git a/examples/standalone_single_project/outputs.tf b/examples/standalone_single_project/outputs.tf new file mode 100644 index 00000000..b07df157 --- /dev/null +++ b/examples/standalone_single_project/outputs.tf @@ -0,0 +1,85 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "cluster_project_id" { + description = "Cluster Project ID" + value = module.multitenant_infra.cluster_project_id +} + +output "cluster_project_number" { + description = "Cluster Project ID" + value = module.multitenant_infra.cluster_project_number +} + +output "network_project_id" { + description = "Network Project ID" + value = module.multitenant_infra.network_project_id +} + +output "fleet_project_id" { + description = "Fleet Project ID" + value = module.multitenant_infra.fleet_project_id +} + +output "env" { + description = "Environment" + value = local.env +} + +output "cluster_regions" { + description = "Regions with clusters" + value = module.multitenant_infra.cluster_regions +} + +output "cluster_membership_ids" { + description = "GKE cluster membership IDs" + value = module.multitenant_infra.cluster_membership_ids +} + +output "app_ip_addresses" { + description = "App IP Addresses" + value = module.multitenant_infra.app_ip_addresses +} + +output "app_certificates" { + description = "App Certificates" + value = module.multitenant_infra.app_certificates +} + +output "acronyms" { + description = "App Acronyms" + value = { for k, v in local.apps : (k) => v.acronym } +} + +output "cluster_type" { + description = "Cluster type" + value = module.multitenant_infra.cluster_type +} + +output "cluster_service_accounts" { + description = "The default service accounts used for nodes, if not overridden in node_pools." + value = module.multitenant_infra.cluster_service_accounts +} + +output "app_infos" { + description = "App infos (name, services, team)." + value = local.cicd_apps +} + +output "clouddeploy_targets_names" { + description = "Cloud deploy targets names." + value = { for k, cicd in module.cicd : k => cicd.clouddeploy_targets_names } +} diff --git a/examples/standalone_single_project/variables.tf b/examples/standalone_single_project/variables.tf new file mode 100644 index 00000000..0c88707f --- /dev/null +++ b/examples/standalone_single_project/variables.tf @@ -0,0 +1,31 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + type = string + description = "Google Cloud project ID in which to deploy all example resources" +} + +variable "region" { + type = string + description = "Google Cloud region for deployments" + default = "us-central1" +} + +variable "teams" { + type = map(string) + description = "A map of string at the format {\"namespace\" = \"groupEmail\"}" +} diff --git a/examples/standalone_single_project/versions.tf b/examples/standalone_single_project/versions.tf new file mode 100644 index 00000000..3ebd1b12 --- /dev/null +++ b/examples/standalone_single_project/versions.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 1.3" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 5, < 7" + } + } + + provider_meta "google" { + module_name = "blueprints/terraform/terraform-google-enterprise-application:standalone-example/v0.2.0" + } +} diff --git a/test/integration/appinfra/appinfra_test.go b/test/integration/appinfra/appinfra_test.go index 4b79133f..cd4a061a 100644 --- a/test/integration/appinfra/appinfra_test.go +++ b/test/integration/appinfra/appinfra_test.go @@ -233,16 +233,9 @@ func TestAppInfra(t *testing.T) { projectRoles = testutils.GetResultFieldStrSlice(filtered, "bindings.role") assert.Subset(projectRoles, cbSARoles, fmt.Sprintf("Service Account %s should have %v roles at project %s.", ciServiceAccountEmail, cbSARoles, servicesInfoMap[fullServiceName].ProjectID)) - cloudDeployTargets := make([]string, 0) - for _, v := range env_cluster_membership_ids { - for _, cluster_membership_id := range v["cluster_membership_ids"] { - cluster_membership_id := strings.Split(cluster_membership_id, "/") - cloudDeployTargets = append(cloudDeployTargets, cluster_membership_id[len(cluster_membership_id)-1]) - } - } - + cloudDeployTargets := appService.GetJsonOutput("clouddeploy_targets_names").Array() for _, targetName := range cloudDeployTargets { - deployTargetOp := gcloud.Runf(t, "deploy targets describe %s --project %s --region %s --flatten Target", strings.TrimPrefix(targetName, "cluster-"), servicesInfoMap[fullServiceName].ProjectID, region).Array()[0] + deployTargetOp := gcloud.Runf(t, "deploy targets describe %s --project %s --region %s --flatten Target", strings.TrimPrefix(targetName.String(), "cluster-"), servicesInfoMap[fullServiceName].ProjectID, region).Array()[0] assert.Equal(cloudDeployServiceAccountEmail, deployTargetOp.Get("executionConfigs").Array()[0].Get("serviceAccount").String(), fmt.Sprintf("cloud deploy target %s should have service account %s", targetName, cloudDeployServiceAccountEmail)) } diff --git a/test/integration/appsource/cymbal_bank_test.go b/test/integration/appsource/cymbal_bank_test.go index 1beac59c..3c73738e 100644 --- a/test/integration/appsource/cymbal_bank_test.go +++ b/test/integration/appsource/cymbal_bank_test.go @@ -36,6 +36,7 @@ import ( func TestSourceCymbalBank(t *testing.T) { env_cluster_membership_ids := make(map[string]map[string][]string, 0) + appName := "cymbal-bank" for _, envName := range testutils.EnvNames(t) { env_cluster_membership_ids[envName] = make(map[string][]string, 0) @@ -56,175 +57,172 @@ func TestSourceCymbalBank(t *testing.T) { region := "us-central1" // TODO: Plumb output from appInfra servicesInfoMap := make(map[string]ServiceInfos) - for appName, serviceNames := range testutils.ServicesNames { - if appName == "cymbal-bank" { - appName := appName - appSourcePath := fmt.Sprintf("../../../examples/%s/6-appsource/%s", appName, appName) - appFactory := tft.NewTFBlueprintTest(t, tft.WithTFDir("../../../4-appfactory/envs/shared")) - for _, serviceName := range serviceNames { - serviceName := serviceName // capture range variable - splitServiceName = strings.Split(serviceName, "-") - prefixServiceName = splitServiceName[0] - suffixServiceName = splitServiceName[len(splitServiceName)-1] - projectID := appFactory.GetJsonOutput("app-group").Get(fmt.Sprintf("%s\\.%s.app_admin_project_id", appName, suffixServiceName)).String() - servicesInfoMap[serviceName] = ServiceInfos{ - ProjectID: projectID, - ServiceName: suffixServiceName, - TeamName: prefixServiceName, - } - servicePath := fmt.Sprintf("%s/%s", appSourcePath, serviceName) - t.Run(servicePath, func(t *testing.T) { - t.Parallel() - mapPath := "" - if servicesInfoMap[serviceName].TeamName == servicesInfoMap[serviceName].ServiceName { - mapPath = servicesInfoMap[serviceName].TeamName - } else { - mapPath = fmt.Sprintf("%s/%s", servicesInfoMap[serviceName].TeamName, servicesInfoMap[serviceName].ServiceName) - } - t.Logf("ServicePath: %s, MapPath: %s", servicePath, mapPath) - appRepo := fmt.Sprintf("https://source.developers.google.com/p/%s/r/eab-%s-%s", servicesInfoMap[serviceName].ProjectID, appName, serviceName) - tmpDirApp := t.TempDir() - dbFrom := fmt.Sprintf("%s/%s-db/k8s/overlays", appSourcePath, servicesInfoMap[serviceName].TeamName) - dbTo := fmt.Sprintf("%s/src/%s/%s-db/k8s/overlays", tmpDirApp, servicesInfoMap[serviceName].TeamName, servicesInfoMap[serviceName].TeamName) - - vars := map[string]interface{}{ - "project_id": servicesInfoMap[serviceName].ProjectID, - "region": region, - "env_cluster_membership_ids": env_cluster_membership_ids, - "buckets_force_destroy": "true", + for _, serviceName := range testutils.ServicesNames[appName] { + appSourcePath := fmt.Sprintf("../../../examples/%s/6-appsource/%s", appName, appName) + appFactory := tft.NewTFBlueprintTest(t, tft.WithTFDir("../../../4-appfactory/envs/shared")) + serviceName := serviceName // capture range variable + splitServiceName = strings.Split(serviceName, "-") + prefixServiceName = splitServiceName[0] + suffixServiceName = splitServiceName[len(splitServiceName)-1] + projectID := appFactory.GetJsonOutput("app-group").Get(fmt.Sprintf("%s\\.%s.app_admin_project_id", appName, suffixServiceName)).String() + appInfra := tft.NewTFBlueprintTest(t, tft.WithTFDir(fmt.Sprintf("../../../examples/%s/5-appinfra/%s/%s/envs/shared", appName, appName, serviceName))) + deployTargets := appInfra.GetJsonOutput("clouddeploy_targets_names") + servicesInfoMap[serviceName] = ServiceInfos{ + ProjectID: projectID, + ServiceName: suffixServiceName, + TeamName: prefixServiceName, + } + servicePath := fmt.Sprintf("%s/%s", appSourcePath, serviceName) + t.Run(servicePath, func(t *testing.T) { + t.Parallel() + mapPath := "" + if servicesInfoMap[serviceName].TeamName == servicesInfoMap[serviceName].ServiceName { + mapPath = servicesInfoMap[serviceName].TeamName + } else { + mapPath = fmt.Sprintf("%s/%s", servicesInfoMap[serviceName].TeamName, servicesInfoMap[serviceName].ServiceName) + } + t.Logf("ServicePath: %s, MapPath: %s", servicePath, mapPath) + appRepo := fmt.Sprintf("https://source.developers.google.com/p/%s/r/eab-%s-%s", servicesInfoMap[serviceName].ProjectID, appName, serviceName) + tmpDirApp := t.TempDir() + dbFrom := fmt.Sprintf("%s/%s-db/k8s/overlays", appSourcePath, servicesInfoMap[serviceName].TeamName) + dbTo := fmt.Sprintf("%s/src/%s/%s-db/k8s/overlays", tmpDirApp, servicesInfoMap[serviceName].TeamName, servicesInfoMap[serviceName].TeamName) + + vars := map[string]interface{}{ + "project_id": servicesInfoMap[serviceName].ProjectID, + "region": region, + "env_cluster_membership_ids": env_cluster_membership_ids, + "buckets_force_destroy": "true", + } + + appsource := tft.NewTFBlueprintTest(t, + tft.WithTFDir(servicePath), + tft.WithVars(vars), + tft.WithRetryableTerraformErrors(testutils.RetryableTransientErrors, 3, 2*time.Minute), + ) + + appsource.DefineVerify(func(assert *assert.Assertions) { + + // Push cymbal bank app source code + gitApp := git.NewCmdConfig(t, git.WithDir(tmpDirApp)) + gitAppRun := func(args ...string) { + _, err := gitApp.RunCmdE(args...) + if err != nil { + t.Fatal(err) } + } - appsource := tft.NewTFBlueprintTest(t, - tft.WithTFDir(servicePath), - tft.WithVars(vars), - tft.WithRetryableTerraformErrors(testutils.RetryableTransientErrors, 3, 2*time.Minute), - ) - - appsource.DefineVerify(func(assert *assert.Assertions) { - - // Push cymbal bank app source code - gitApp := git.NewCmdConfig(t, git.WithDir(tmpDirApp)) - gitAppRun := func(args ...string) { - _, err := gitApp.RunCmdE(args...) - if err != nil { - t.Fatal(err) - } - } + gitAppRun("clone", "--branch", "v0.6.4", "https://github.com/GoogleCloudPlatform/bank-of-anthos.git", tmpDirApp) + gitAppRun("config", "user.email", "eab-robot@example.com") + gitAppRun("config", "user.name", "EAB Robot") + gitAppRun("config", "credential.https://source.developers.google.com.helper", "gcloud.sh") + gitAppRun("config", "init.defaultBranch", "main") + gitAppRun("config", "http.postBuffer", "157286400") + gitAppRun("checkout", "-b", "main") + gitAppRun("remote", "add", "google", appRepo) + datefile, err := os.OpenFile(fmt.Sprintf("%s/src/%s/date.txt", tmpDirApp, mapPath), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + t.Fatal(err) + } + defer datefile.Close() - gitAppRun("clone", "--branch", "v0.6.4", "https://github.com/GoogleCloudPlatform/bank-of-anthos.git", tmpDirApp) - gitAppRun("config", "user.email", "eab-robot@example.com") - gitAppRun("config", "user.name", "EAB Robot") - gitAppRun("config", "credential.https://source.developers.google.com.helper", "gcloud.sh") - gitAppRun("config", "init.defaultBranch", "main") - gitAppRun("config", "http.postBuffer", "157286400") - gitAppRun("checkout", "-b", "main") - gitAppRun("remote", "add", "google", appRepo) - datefile, err := os.OpenFile(fmt.Sprintf("%s/src/%s/date.txt", tmpDirApp, mapPath), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - t.Fatal(err) - } - defer datefile.Close() + _, err = datefile.WriteString(time.Now().String() + "\n") + if err != nil { + t.Fatal(err) + } + gitAppRun("rm", "-r", "src/components") + + // MapPaths which will get the database overlay + dbPaths := []string{"accounts/contacts", "ledger/balancereader"} + + // base folder only exists in frontend app + if mapPath == "frontend" { + gitAppRun("rm", "-r", fmt.Sprintf("src/%s/k8s", mapPath)) + } else if slices.Contains(dbPaths, mapPath) { + t.Logf("%s - Copying from %s to %s", servicePath, dbFrom, dbTo) + err = cp.Copy(dbFrom, dbTo) + if err != nil { + t.Fatal(err) + } + } else { + t.Logf("%s - Removing database %s", servicePath, dbTo) + gitAppRun("rm", "-r", dbTo) + } + err = cp.Copy(fmt.Sprintf("%s/components", appSourcePath), fmt.Sprintf("%s/src/components", tmpDirApp)) + if err != nil { + t.Fatal(err) + } + err = cp.Copy(fmt.Sprintf("%s/skaffold.yaml", servicePath), fmt.Sprintf("%s/src/%s/skaffold.yaml", tmpDirApp, mapPath)) + if err != nil { + t.Fatal(err) + } + err = cp.Copy(fmt.Sprintf("%s/k8s", servicePath), fmt.Sprintf("%s/src/%s/k8s", tmpDirApp, mapPath)) + if err != nil { + t.Fatal(err) + } - _, err = datefile.WriteString(time.Now().String() + "\n") - if err != nil { - t.Fatal(err) - } - gitAppRun("rm", "-r", "src/components") - - // MapPaths which will get the database overlay - dbPaths := []string{"accounts/contacts", "ledger/balancereader"} - - // base folder only exists in frontend app - if mapPath == "frontend" { - gitAppRun("rm", "-r", fmt.Sprintf("src/%s/k8s", mapPath)) - } else if slices.Contains(dbPaths, mapPath) { - t.Logf("%s - Copying from %s to %s", servicePath, dbFrom, dbTo) - err = cp.Copy(dbFrom, dbTo) - if err != nil { - t.Fatal(err) - } - } else { - t.Logf("%s - Removing database %s", servicePath, dbTo) - gitAppRun("rm", "-r", dbTo) - } - err = cp.Copy(fmt.Sprintf("%s/components", appSourcePath), fmt.Sprintf("%s/src/components", tmpDirApp)) - if err != nil { - t.Fatal(err) - } - err = cp.Copy(fmt.Sprintf("%s/skaffold.yaml", servicePath), fmt.Sprintf("%s/src/%s/skaffold.yaml", tmpDirApp, mapPath)) - if err != nil { - t.Fatal(err) - } - err = cp.Copy(fmt.Sprintf("%s/k8s", servicePath), fmt.Sprintf("%s/src/%s/k8s", tmpDirApp, mapPath)) - if err != nil { - t.Fatal(err) - } + // Copy test-specific k8s manifests to the frontend development overlay + if mapPath == "frontend" { + err = cp.Copy("assets/", fmt.Sprintf("%s/src/%s/k8s/overlays/development/", tmpDirApp, mapPath)) + if err != nil { + t.Fatal(err) + } + } - // Copy test-specific k8s manifests to the frontend development overlay - if mapPath == "frontend" { - err = cp.Copy("assets/", fmt.Sprintf("%s/src/%s/k8s/overlays/development/", tmpDirApp, mapPath)) - if err != nil { - t.Fatal(err) - } + gitAppRun("add", ".") + gitApp.CommitWithMsg("initial commit", []string{"--allow-empty"}) + gitAppRun("push", "--all", "google", "-f") + + lastCommit := gitApp.GetLatestCommit() + // filter builds triggered based on pushed commit sha + buildListCmd := fmt.Sprintf("builds list --region=%s --filter substitutions.COMMIT_SHA='%s' --project %s", region, lastCommit, servicesInfoMap[serviceName].ProjectID) + // poll build until complete + pollCloudBuild := func(cmd string) func() (bool, error) { + return func() (bool, error) { + build := gcloud.Runf(t, cmd).Array() + if len(build) < 1 { + return true, nil } - - gitAppRun("add", ".") - gitApp.CommitWithMsg("initial commit", []string{"--allow-empty"}) - gitAppRun("push", "--all", "google", "-f") - - lastCommit := gitApp.GetLatestCommit() - // filter builds triggered based on pushed commit sha - buildListCmd := fmt.Sprintf("builds list --region=%s --filter substitutions.COMMIT_SHA='%s' --project %s", region, lastCommit, servicesInfoMap[serviceName].ProjectID) - // poll build until complete - pollCloudBuild := func(cmd string) func() (bool, error) { - return func() (bool, error) { - build := gcloud.Runf(t, cmd).Array() - if len(build) < 1 { - return true, nil - } - latestWorkflowRunStatus := build[0].Get("status").String() - if latestWorkflowRunStatus == "SUCCESS" { - return false, nil - } else if latestWorkflowRunStatus == "FAILURE" { - return false, errors.New("Build failed.") - } - return true, nil - } + latestWorkflowRunStatus := build[0].Get("status").String() + if latestWorkflowRunStatus == "SUCCESS" { + return false, nil + } else if latestWorkflowRunStatus == "FAILURE" { + return false, errors.New("Build failed.") } - utils.Poll(t, pollCloudBuild(buildListCmd), 40, 30*time.Second) - releaseListCmd := fmt.Sprintf("deploy releases list --project=%s --delivery-pipeline=%s --region=%s --filter=name:%s", servicesInfoMap[serviceName].ProjectID, servicesInfoMap[serviceName].ServiceName, region, lastCommit[0:7]) - releases := gcloud.Runf(t, releaseListCmd).Array() - if len(releases) == 0 { - t.Fatal("Failed to find the release.") + return true, nil + } + } + utils.Poll(t, pollCloudBuild(buildListCmd), 40, 60*time.Second) + releaseListCmd := fmt.Sprintf("deploy releases list --project=%s --delivery-pipeline=%s --region=%s --filter=name:%s", servicesInfoMap[serviceName].ProjectID, servicesInfoMap[serviceName].ServiceName, region, lastCommit[0:7]) + releases := gcloud.Runf(t, releaseListCmd).Array() + if len(releases) == 0 { + t.Fatal("Failed to find the release.") + } + releaseName := releases[0].Get("name") + targetId := deployTargets.Array()[0] + rolloutListCmd := fmt.Sprintf("deploy rollouts list --project=%s --delivery-pipeline=%s --region=%s --release=%s --filter targetId=%s", servicesInfoMap[serviceName].ProjectID, servicesInfoMap[serviceName].ServiceName, region, releaseName, targetId) + // Poll CD rollouts until rollout is successful + pollCloudDeploy := func(cmd string) func() (bool, error) { + return func() (bool, error) { + rollouts := gcloud.Runf(t, cmd).Array() + if len(rollouts) < 1 { + return true, nil } - releaseName := releases[0].Get("name") - targetId := fmt.Sprintf("%s-development", region) //TODO: convert to loop using env_cluster_membership_ids - rolloutListCmd := fmt.Sprintf("deploy rollouts list --project=%s --delivery-pipeline=%s --region=%s --release=%s --filter targetId=%s", servicesInfoMap[serviceName].ProjectID, servicesInfoMap[serviceName].ServiceName, region, releaseName, targetId) - // Poll CD rollouts until rollout is successful - pollCloudDeploy := func(cmd string) func() (bool, error) { - return func() (bool, error) { - rollouts := gcloud.Runf(t, cmd).Array() - if len(rollouts) < 1 { - return true, nil - } - latestRolloutState := rollouts[0].Get("state").String() - if latestRolloutState == "SUCCEEDED" { - return false, nil - } else if slices.Contains([]string{"IN_PROGRESS", "PENDING_RELEASE"}, latestRolloutState) { - return true, nil - } else { - logsCmd := fmt.Sprintf("builds log %s", rollouts[0].Get("deployingBuild").String()) - logs := gcloud.Runf(t, logsCmd).String() - t.Logf("%s build-log: %s", servicesInfoMap[serviceName].ServiceName, logs) - return false, fmt.Errorf("Rollout %s.", latestRolloutState) - } - } + latestRolloutState := rollouts[0].Get("state").String() + if latestRolloutState == "SUCCEEDED" { + return false, nil + } else if slices.Contains([]string{"IN_PROGRESS", "PENDING_RELEASE"}, latestRolloutState) { + return true, nil + } else { + logsCmd := fmt.Sprintf("builds log %s", rollouts[0].Get("deployingBuild").String()) + logs := gcloud.Runf(t, logsCmd).String() + t.Logf("%s build-log: %s", servicesInfoMap[serviceName].ServiceName, logs) + return false, fmt.Errorf("Rollout %s.", latestRolloutState) } - utils.Poll(t, pollCloudDeploy(rolloutListCmd), 30, 60*time.Second) - }) - appsource.Test() - }) - } - } + } + } + utils.Poll(t, pollCloudDeploy(rolloutListCmd), 30, 60*time.Second) + }) + appsource.Test() + }) } } diff --git a/test/integration/appsource/cymbal_shop_test.go b/test/integration/appsource/cymbal_shop_test.go index 66859a9c..39fd0559 100644 --- a/test/integration/appsource/cymbal_shop_test.go +++ b/test/integration/appsource/cymbal_shop_test.go @@ -49,6 +49,8 @@ func TestSourceCymbalShop(t *testing.T) { appFactory := tft.NewTFBlueprintTest(t, tft.WithTFDir("../../../4-appfactory/envs/shared")) projectID := appFactory.GetJsonOutput("app-group").Get("cymbal-shop\\.cymbalshop.app_admin_project_id").String() + appInfra := tft.NewTFBlueprintTest(t, tft.WithTFDir(fmt.Sprintf("../../../examples/%s/5-appinfra/%s/%s/envs/shared", appName, appName, serviceName))) + deployTargets := appInfra.GetJsonOutput("clouddeploy_targets_names") t.Run("replace-repo-contents-and-push", func(t *testing.T) { @@ -126,7 +128,7 @@ func TestSourceCymbalShop(t *testing.T) { t.Fatal("Failed to find the release.") } releaseName := releases[0].Get("name") - targetId := fmt.Sprintf("%s-development", region) //TODO: convert to loop using env_cluster_membership_ids + targetId := deployTargets.Array()[0] rolloutListCmd := fmt.Sprintf("deploy rollouts list --project=%s --delivery-pipeline=%s --region=%s --release=%s --filter targetId=%s", projectID, serviceName, region, releaseName, targetId) // Poll CD rollouts until rollout is successful pollCloudDeploy := func(cmd string) func() (bool, error) { @@ -137,8 +139,10 @@ func TestSourceCymbalShop(t *testing.T) { } latestRolloutState := rollouts[0].Get("state").String() if latestRolloutState == "SUCCEEDED" { + t.Logf("Rollout finished successfully %s. \n", rollouts[0].Get("targetId")) return false, nil } else if slices.Contains([]string{"IN_PROGRESS", "PENDING_RELEASE"}, latestRolloutState) { + t.Logf("Rollout in progress %s. \n", rollouts[0].Get("targetId")) return true, nil } else { logsCmd := fmt.Sprintf("builds log %s", rollouts[0].Get("deployingBuild").String()) diff --git a/test/integration/cymbal-bank/e2e_single_project_test.go b/test/integration/cymbal-bank/e2e_single_project_test.go new file mode 100644 index 00000000..80a0240f --- /dev/null +++ b/test/integration/cymbal-bank/e2e_single_project_test.go @@ -0,0 +1,143 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cymbalbank_e2e + +import ( + "context" + "fmt" + "net/http" + "net/http/cookiejar" + "testing" + + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" + "github.com/gruntwork-io/terratest/modules/retry" +) + +func TestAppE2ECymbalBankSingleProject(t *testing.T) { + // initialize Terraform test from the Blueprints test framework + setupOutput := tft.NewTFBlueprintTest(t) + projectID := setupOutput.GetTFSetupStringOutput("project_id_standalone") + standaloneSingleProj := tft.NewTFBlueprintTest(t, tft.WithVars(map[string]interface{}{"project_id": projectID}), tft.WithTFDir("../../../examples/standalone_single_project")) + t.Run("End to end tests Single project", func(t *testing.T) { + jar, err := cookiejar.New(nil) + if err != nil { + t.Fatal(err) + } + client := &http.Client{ + Jar: jar, + } + ctx := context.Background() + ipAddress := standaloneSingleProj.GetJsonOutput("app_ip_addresses").Get("cymbal-bank.cb-frontend-ip").String() + + // Test webserver is avaliable + heartbeat := func() (string, error) { + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("http://%s", ipAddress), nil) + if err != nil { + return "", err + } + resp, err := client.Do(req) + if err != nil { + return "", err + } + if resp.StatusCode != 200 { + fmt.Println(resp) + return "", err + } + return fmt.Sprint(resp.StatusCode), err + } + statusCode, _ := retry.DoWithRetryE( + t, + fmt.Sprintf("Checking: %s", ipAddress), + maxRetries, + sleepBetweenRetries, + heartbeat, + ) + if err != nil { + t.Fatalf("Error: webserver (%s) not ready after %d attemps, status code: %q", + ipAddress, + maxRetries, + statusCode, + ) + } + + resp, err := login(ctx, client, ipAddress) + + // check if the server replied with authentication cookie + gotToken := false + for _, cookie := range resp.Request.Cookies() { + if cookie.Name == "token" { + gotToken = true + } + } + + if resp.StatusCode != 200 { + fmt.Println(resp) + t.Fatal(err) + } + + if !gotToken { + fmt.Println("Failed Authentication.") + fmt.Println(resp.Request) + t.Fatal(err) + } + + fmt.Printf("Login resp: %v \n", resp) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != 200 { + fmt.Println(resp) + t.Fatal(err) + } + + resp, err = home(ctx, client, ipAddress) + fmt.Printf("Home resp: %v \n", resp) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != 200 { + fmt.Println(resp) + t.Fatal(err) + } + + resp, err = deposit(ctx, client, ipAddress) + fmt.Printf("Deposit resp: %v \n", resp) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != 200 { + fmt.Println(resp) + t.Fatal(err) + } + + _, err = checkTransaction(ctx, client, ipAddress, "5,230.00", "DEPOSIT") + if err != nil { + t.Fatal(err) + } + resp, err = transfer(ctx, client, ipAddress) + fmt.Printf("Transfer resp: %v \n", resp) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != 200 { + fmt.Println(resp) + t.Fatal(err) + } + _, err = checkTransaction(ctx, client, ipAddress, "320.98", "PAYMENT") + if err != nil { + t.Fatal(err) + } + }) +} diff --git a/test/integration/cymbal-bank/e2e_test.go b/test/integration/cymbal-bank/e2e_test.go index 4231dce8..4ec391e8 100644 --- a/test/integration/cymbal-bank/e2e_test.go +++ b/test/integration/cymbal-bank/e2e_test.go @@ -38,7 +38,7 @@ const ( func TestCymbalBankE2E(t *testing.T) { multitenant := tft.NewTFBlueprintTest(t, tft.WithTFDir("../../../2-multitenant/envs/development")) - t.Run("End to end tests", func(t *testing.T) { + t.Run("End to end tests Cymbal Bank Multitenant", func(t *testing.T) { jar, err := cookiejar.New(nil) if err != nil { t.Fatal(err) diff --git a/test/integration/fleetscope/fleetscope_test.go b/test/integration/fleetscope/fleetscope_test.go index 5a44e7ef..832b9a0f 100644 --- a/test/integration/fleetscope/fleetscope_test.go +++ b/test/integration/fleetscope/fleetscope_test.go @@ -182,11 +182,10 @@ func TestFleetscope(t *testing.T) { controlPlaneManagement := result.Get("membershipStates").Get(memberShipName).Get("servicemesh.controlPlaneManagement.state").String() if dataPlaneManagement == "PROVISIONING" || controlPlaneManagement == "PROVISIONING" { retry = true - } else if (dataPlaneManagement == "ACTIVE" && controlPlaneManagement == "ACTIVE") && !retry { - // if there is no other membership still in PROVISIONING - retry = false - } else { - return false, fmt.Errorf("Service mesh provisioning failed for %s: dataPlaneManagement = %s and controlPlaneManagement = %s", memberShipName, dataPlaneManagement, controlPlaneManagement) + } else if !(dataPlaneManagement == "ACTIVE" && controlPlaneManagement == "ACTIVE") { + generalState := result.Get("membershipStates").Get(memberShipName).Get("state.code").String() + generalDescription := result.Get("membershipStates").Get(memberShipName).Get("state.description").String() + return false, fmt.Errorf("Service mesh provisioning failed for %s: status='%s' description='%s'", memberShipName, generalState, generalDescription) } } return retry, nil diff --git a/test/integration/multitenant/multitenant_test.go b/test/integration/multitenant/multitenant_test.go index 35893db4..92d6f2f1 100644 --- a/test/integration/multitenant/multitenant_test.go +++ b/test/integration/multitenant/multitenant_test.go @@ -207,6 +207,22 @@ func TestMultitenant(t *testing.T) { } }) + multitenant.DefineTeardown(func(assert *assert.Assertions) { + multitenant := tft.NewTFBlueprintTest(t, + tft.WithTFDir(fmt.Sprintf("../../../2-multitenant/envs/%s", envName)), + tft.WithRetryableTerraformErrors(testutils.RetryableTransientErrors, 3, 2*time.Minute), + tft.WithBackendConfig(backendConfig), + ) + clusterProjectID := multitenant.GetStringOutput("cluster_project_id") + // removes firewall rules created by the service but not being deleted. + firewallRules := gcloud.Runf(t, "compute firewall-rules list --project %s --filter=\"mcsd\"", clusterProjectID).Array() + for i := range firewallRules { + gcloud.Runf(t, "compute firewall-rules delete %s --project %s -q", firewallRules[i].Get("name"), clusterProjectID) + } + multitenant.DefaultTeardown(assert) + + }) + multitenant.Test() }) } diff --git a/test/integration/standalone_single_project/cymbal_bank_single_project_test.go b/test/integration/standalone_single_project/cymbal_bank_single_project_test.go new file mode 100644 index 00000000..e5c3f805 --- /dev/null +++ b/test/integration/standalone_single_project/cymbal_bank_single_project_test.go @@ -0,0 +1,229 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package standalone_single_project + +import ( + "fmt" + "os" + "slices" + "strings" + "testing" + "time" + + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/git" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils" + "github.com/stretchr/testify/assert" + "github.com/terraform-google-modules/enterprise-application/test/integration/testutils" + + cp "github.com/otiai10/copy" +) + +func TestSingleProjectSourceCymbalBank(t *testing.T) { + + env_cluster_membership_ids := make(map[string]map[string][]string, 0) + // initialize Terraform test from the Blueprints test framework + setupOutput := tft.NewTFBlueprintTest(t) + projectID := setupOutput.GetTFSetupStringOutput("project_id_standalone") + standaloneSingleProj := tft.NewTFBlueprintTest(t, tft.WithVars(map[string]interface{}{"project_id": projectID}), tft.WithTFDir("../../../examples/standalone_single_project")) + envName := standaloneSingleProj.GetStringOutput("env") + env_cluster_membership_ids[envName] = make(map[string][]string, 0) + env_cluster_membership_ids[envName]["cluster_membership_ids"] = testutils.GetBptOutputStrSlice(standaloneSingleProj, "cluster_membership_ids") + deployTargets := standaloneSingleProj.GetJsonOutput("clouddeploy_targets_names") + + type ServiceInfos struct { + ProjectID string + ServiceName string + TeamName string + } + var ( + prefixServiceName string + suffixServiceName string + splitServiceName []string + ) + region := "us-central1" + servicesInfoMap := make(map[string]ServiceInfos) + appName := "cymbal-bank" + appSourcePath := fmt.Sprintf("../../../examples/%s/6-appsource/%s", appName, appName) + for _, serviceName := range testutils.ServicesNames[appName] { + serviceName := serviceName // capture range variable + splitServiceName = strings.Split(serviceName, "-") + prefixServiceName = splitServiceName[0] + suffixServiceName = splitServiceName[len(splitServiceName)-1] + servicesInfoMap[serviceName] = ServiceInfos{ + ProjectID: projectID, + ServiceName: suffixServiceName, + TeamName: prefixServiceName, + } + servicePath := fmt.Sprintf("%s/%s", appSourcePath, serviceName) + t.Log(servicePath) + t.Run(servicePath, func(t *testing.T) { + t.Parallel() + vars := map[string]interface{}{ + "project_id": servicesInfoMap[serviceName].ProjectID, + "region": region, + "env_cluster_membership_ids": env_cluster_membership_ids, + "buckets_force_destroy": "true", + } + + appsource := tft.NewTFBlueprintTest(t, + tft.WithTFDir(servicePath), + tft.WithVars(vars), + tft.WithRetryableTerraformErrors(testutils.RetryableTransientErrors, 3, 2*time.Minute), + ) + mapPath := "" + if servicesInfoMap[serviceName].TeamName == servicesInfoMap[serviceName].ServiceName { + mapPath = servicesInfoMap[serviceName].TeamName + } else { + mapPath = fmt.Sprintf("%s/%s", servicesInfoMap[serviceName].TeamName, servicesInfoMap[serviceName].ServiceName) + } + t.Logf("ServicePath: %s, MapPath: %s", servicePath, mapPath) + appRepo := fmt.Sprintf("https://source.developers.google.com/p/%s/r/eab-%s-%s", servicesInfoMap[serviceName].ProjectID, appName, serviceName) + tmpDirApp := t.TempDir() + dbFrom := fmt.Sprintf("%s/%s-db/k8s/overlays", appSourcePath, servicesInfoMap[serviceName].TeamName) + dbTo := fmt.Sprintf("%s/src/%s/%s-db/k8s/overlays", tmpDirApp, servicesInfoMap[serviceName].TeamName, servicesInfoMap[serviceName].TeamName) + + appsource.DefineVerify(func(assert *assert.Assertions) { + + // Push cymbal bank app source code + gitApp := git.NewCmdConfig(t, git.WithDir(tmpDirApp)) + gitAppRun := func(args ...string) { + _, err := gitApp.RunCmdE(args...) + if err != nil { + t.Fatal(err) + } + } + + gitAppRun("clone", "--branch", "v0.6.4", "https://github.com/GoogleCloudPlatform/bank-of-anthos.git", tmpDirApp) + gitAppRun("config", "user.email", "eab-robot@example.com") + gitAppRun("config", "user.name", "EAB Robot") + gitAppRun("config", "credential.https://source.developers.google.com.helper", "gcloud.sh") + gitAppRun("config", "init.defaultBranch", "main") + gitAppRun("config", "http.postBuffer", "157286400") + gitAppRun("checkout", "-b", "main") + gitAppRun("remote", "add", "google", appRepo) + datefile, err := os.OpenFile(fmt.Sprintf("%s/src/%s/date.txt", tmpDirApp, mapPath), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + t.Fatal(err) + } + defer datefile.Close() + + _, err = datefile.WriteString(time.Now().String() + "\n") + if err != nil { + t.Fatal(err) + } + gitAppRun("rm", "-r", "src/components") + + // MapPaths which will get the database overlay + dbPaths := []string{"accounts/contacts", "ledger/balancereader"} + + // base folder only exists in frontend app + if mapPath == "frontend" { + gitAppRun("rm", "-r", fmt.Sprintf("src/%s/k8s", mapPath)) + } else if slices.Contains(dbPaths, mapPath) { + t.Logf("%s - Copying from %s to %s", servicePath, dbFrom, dbTo) + err = cp.Copy(dbFrom, dbTo) + if err != nil { + t.Fatal(err) + } + } else { + t.Logf("%s - Removing database %s", servicePath, dbTo) + gitAppRun("rm", "-r", dbTo) + } + err = cp.Copy(fmt.Sprintf("%s/components", appSourcePath), fmt.Sprintf("%s/src/components", tmpDirApp)) + if err != nil { + t.Fatal(err) + } + err = cp.Copy(fmt.Sprintf("%s/skaffold.yaml", servicePath), fmt.Sprintf("%s/src/%s/skaffold.yaml", tmpDirApp, mapPath)) + if err != nil { + t.Fatal(err) + } + err = cp.Copy(fmt.Sprintf("%s/k8s", servicePath), fmt.Sprintf("%s/src/%s/k8s", tmpDirApp, mapPath)) + if err != nil { + t.Fatal(err) + } + + // Copy test-specific k8s manifests to the frontend development overlay + if mapPath == "frontend" { + err = cp.Copy("../appsource/assets/", fmt.Sprintf("%s/src/%s/k8s/overlays/development/", tmpDirApp, mapPath)) + if err != nil { + t.Fatal(err) + } + } + + gitAppRun("add", ".") + gitApp.CommitWithMsg("initial commit", []string{"--allow-empty"}) + gitAppRun("push", "--all", "google", "-f") + + lastCommit := gitApp.GetLatestCommit() + // filter builds triggered based on pushed commit sha + buildListCmd := fmt.Sprintf("builds list --region=%s --filter substitutions.COMMIT_SHA='%s' --project %s", region, lastCommit, servicesInfoMap[serviceName].ProjectID) + // poll build until complete + pollCloudBuild := func(cmd string) func() (bool, error) { + return func() (bool, error) { + build := gcloud.Runf(t, cmd).Array() + if len(build) < 1 { + return true, nil + } + latestWorkflowRunStatus := build[0].Get("status").String() + t.Logf("Build found for commit %s: %s \n", lastCommit, build[0].Get("buildTriggerId")) + if latestWorkflowRunStatus == "SUCCESS" { + t.Logf("Build finished successfully %s. \n", build[0].Get("buildTriggerId")) + return false, nil + } else if latestWorkflowRunStatus == "FAILURE" { + return false, fmt.Errorf("Build failed %s.", build[0].Get("buildTriggerId")) + } + return true, nil + } + } + utils.Poll(t, pollCloudBuild(buildListCmd), 40, 60*time.Second) + releaseListCmd := fmt.Sprintf("deploy releases list --project=%s --delivery-pipeline=%s --region=%s --filter=name:%s", servicesInfoMap[serviceName].ProjectID, servicesInfoMap[serviceName].ServiceName, region, lastCommit[0:7]) + releases := gcloud.Runf(t, releaseListCmd).Array() + if len(releases) == 0 { + t.Fatal("Failed to find the release.") + } + releaseName := releases[0].Get("name") + targetId := deployTargets.Get(servicesInfoMap[serviceName].ServiceName).Array()[0] + rolloutListCmd := fmt.Sprintf("deploy rollouts list --project=%s --delivery-pipeline=%s --region=%s --release=%s --filter targetId=%s", servicesInfoMap[serviceName].ProjectID, servicesInfoMap[serviceName].ServiceName, region, releaseName, targetId) + // Poll CD rollouts until rollout is successful + pollCloudDeploy := func(cmd string) func() (bool, error) { + return func() (bool, error) { + rollouts := gcloud.Runf(t, cmd).Array() + if len(rollouts) < 1 { + return true, nil + } + latestRolloutState := rollouts[0].Get("state").String() + if latestRolloutState == "SUCCEEDED" { + t.Logf("Rollout finished successfully %s. \n", rollouts[0].Get("targetId")) + return false, nil + } else if slices.Contains([]string{"IN_PROGRESS", "PENDING_RELEASE"}, latestRolloutState) { + t.Logf("Rollout in progress %s. \n", rollouts[0].Get("targetId")) + return true, nil + } else { + logsCmd := fmt.Sprintf("builds log %s", rollouts[0].Get("deployingBuild").String()) + logs := gcloud.Runf(t, logsCmd).String() + t.Logf("%s build-log: %s", servicesInfoMap[serviceName].ServiceName, logs) + return false, fmt.Errorf("Rollout %s.", latestRolloutState) + } + } + } + utils.Poll(t, pollCloudDeploy(rolloutListCmd), 40, 60*time.Second) + }) + appsource.Test() + }) + + } +} diff --git a/test/integration/standalone_single_project/standalone_single_project_test.go b/test/integration/standalone_single_project/standalone_single_project_test.go new file mode 100644 index 00000000..53cbf5e0 --- /dev/null +++ b/test/integration/standalone_single_project/standalone_single_project_test.go @@ -0,0 +1,195 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// define test package name +package standalone_single_project + +import ( + "fmt" + "net" + "regexp" + "strings" + "testing" + "time" + + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils" + "github.com/stretchr/testify/assert" + "github.com/tidwall/gjson" + + "github.com/terraform-google-modules/enterprise-application/test/integration/testutils" +) + +// name the function as Test* +func TestStandaloneSingleProjectExample(t *testing.T) { + + // initialize Terraform test from the Blueprints test framework + setupOutput := tft.NewTFBlueprintTest(t) + projectID := setupOutput.GetTFSetupStringOutput("project_id_standalone") + + // wire setup output project_id_standalone to example var.project_id + standaloneSingleProjT := tft.NewTFBlueprintTest(t, + tft.WithVars(map[string]interface{}{"project_id": projectID}), + tft.WithTFDir("../../../examples/standalone_single_project"), + tft.WithRetryableTerraformErrors(testutils.RetryableTransientErrors, 3, 2*time.Minute), + ) + + // define and write a custom verifier for this test case call the default verify for confirming no additional changes + standaloneSingleProjT.DefineVerify(func(assert *assert.Assertions) { + // perform default verification ensuring Terraform reports no additional changes on an applied blueprint + // standaloneSingleProjT.DefaultVerify(assert) + clusterMembershipIds := testutils.GetBptOutputStrSlice(standaloneSingleProjT, "cluster_membership_ids") + clusterType := standaloneSingleProjT.GetStringOutput("cluster_type") + clusterProjectNumber := standaloneSingleProjT.GetStringOutput("cluster_project_number") + clusterRegions := testutils.GetBptOutputStrSlice(standaloneSingleProjT, "cluster_regions") + envName := standaloneSingleProjT.GetStringOutput("env") + listMonitoringEnabledComponents := []string{ + "SYSTEM_COMPONENTS", + "DEPLOYMENT", + } + + for _, id := range clusterMembershipIds { + // Membership details + membershipOp := gcloud.Runf(t, "container fleet memberships describe %s", strings.TrimPrefix(id, "//gkehub.googleapis.com/")) + // Cluster details + clusterLocation := regexp.MustCompile(`\/locations\/([^\/]*)\/`).FindStringSubmatch(membershipOp.Get("endpoint.gkeCluster.resourceLink").String())[1] + clusterName := regexp.MustCompile(`\/clusters\/([^\/]*)$`).FindStringSubmatch(membershipOp.Get("endpoint.gkeCluster.resourceLink").String())[1] + clusterOp := gcloud.Runf(t, "container clusters describe %s --location %s --project %s", clusterName, clusterLocation, projectID) + + // Extract enablePrivateEndpoint flag value + enablePrivateEndpoint := clusterOp.Get("privateClusterConfig.enablePrivateEndpoint").Bool() + assert.True(enablePrivateEndpoint, "The cluster external endpoint must be private.") + + // Validate if all nodes inside node pool does not contain an external NAT IP address + nodePoolName := clusterOp.Get("nodePools.0.name").String() + nodeInstances := gcloud.Runf(t, "compute instances list --filter=\"labels.goog-k8s-node-pool-name=%s\" --project=%s", nodePoolName, projectID).Array() + for _, node := range nodeInstances { + // retrieve all node network interfaces + nics := node.Get("networkInterfaces") + // for each network interface, verify if it using an external natIP + nics.ForEach((func(key, value gjson.Result) bool { + assert.Equal(nil, net.ParseIP(value.Get("accessConfigs.0.natIP").String()), "The nodes inside the nodepool should not have external ip addresses.") + return true // keep iterating + })) + } + // NodePools + switch clusterType { + case "STANDARD": + assert.Equal("node-pool-1", clusterOp.Get("nodePools.0.name").String(), "NodePool name should be node-pool-1") + assert.Equal("SURGE", clusterOp.Get("nodePools.0.upgradeSettings.strategy").String(), "NodePool strategy should SURGE") + assert.Equal("1", clusterOp.Get("nodePools.0.upgradeSettings.maxSurge").String(), "NodePool max surge should be 1") + assert.Equal("BALANCED", clusterOp.Get("nodePools.0.autoscaling.locationPolicy").String(), "NodePool auto scaling location prolicy should be BALANCED") + assert.True(clusterOp.Get("nodePools.0.autoscaling.enabled").Bool(), "NodePool auto scaling should be enabled (true)") + case "STANDARD-NAP": + for _, pool := range clusterOp.Get("nodePools").Array() { + if pool.Get("name").String() == "node-pool-1" { + assert.False(pool.Get("autoscaling.autoprovisioned").Bool(), "NodePool autoscaling autoprovisioned should disabled(false)") + } else if regexp.MustCompile(`^nap-.*`).FindString(pool.Get("name").String()) != "" { + assert.True(pool.Get("autoscaling.autoprovisioned").Bool(), "NodePool autoscaling autoprovisioned should enabled(true)") + } else { + t.Fatalf("Error: unknown node pool: %s", pool.Get("name").String()) + } + // common to all valid node pools + assert.True(pool.Get("autoscaling.enabled").Bool(), "NodePool auto scaling should be enabled (true)") + assert.Equal("SURGE", pool.Get("upgradeSettings.strategy").String(), "NodePool strategy should SURGE") + assert.Equal("1", pool.Get("upgradeSettings.maxSurge").String(), "NodePool max surge should be 1") + assert.Equal("BALANCED", pool.Get("autoscaling.locationPolicy").String(), "NodePool auto scaling location prolicy should be BALANCED") + } + case "AUTOPILOT": + // Autopilot manages all nodepools + default: + t.Fatalf("Error: unknown cluster type: %s", clusterType) + } + // Cluster + assert.Equal(projectID, clusterOp.Get("fleet.project").String(), fmt.Sprintf("Cluster %s Fleet Project should be %s", id, projectID)) + clusterEnabledComponents := utils.GetResultStrSlice(clusterOp.Get("monitoringConfig.componentConfig.enableComponents").Array()) + if clusterType != "AUTOPILOT" { + assert.Equal(listMonitoringEnabledComponents, clusterEnabledComponents, fmt.Sprintf("Cluster %s should have Monitoring Enabled Components: SYSTEM_COMPONENTS and DEPLOYMENT", id)) + } + assert.True(clusterOp.Get("monitoringConfig.managedPrometheusConfig.enabled").Bool(), fmt.Sprintf("Cluster %s should have Managed Prometheus Config equals True", id)) + assert.Equal(fmt.Sprintf("%s.svc.id.goog", projectID), clusterOp.Get("workloadIdentityConfig.workloadPool").String(), fmt.Sprintf("Cluster %s workloadPool should be %s.svc.id.goog", id, projectID)) + assert.Equal(fmt.Sprintf("%s.svc.id.goog", projectID), membershipOp.Get("authority.workloadIdentityPool").String(), fmt.Sprintf("Membership %s workloadIdentityPool should be %s.svc.id.goog", id, projectID)) + assert.Equal("PROJECT_SINGLETON_POLICY_ENFORCE", clusterOp.Get("binaryAuthorization.evaluationMode").String(), fmt.Sprintf("Cluster %s Binary Authorization Evaluation Mode should be PROJECT_SINGLETON_POLICY_ENFORCE", id)) + + } + + // Service Identity + fleetProjectNumber := gcloud.Runf(t, "projects describe %s", projectID).Get("projectNumber").String() + gkeServiceAgent := fmt.Sprintf("service-%s@gcp-sa-gkehub.iam.gserviceaccount.com", fleetProjectNumber) + gkeSaRoles := []string{"roles/gkehub.serviceAgent"} + + gkeIamFilter := fmt.Sprintf("bindings.members:'serviceAccount:%s'", gkeServiceAgent) + gkeIamCommonArgs := gcloud.WithCommonArgs([]string{"--flatten", "bindings", "--filter", gkeIamFilter, "--format", "json"}) + gkeProjectPolicyOp := gcloud.Run(t, fmt.Sprintf("projects get-iam-policy %s", projectID), gkeIamCommonArgs).Array() + gkeSaListRoles := testutils.GetResultFieldStrSlice(gkeProjectPolicyOp, "bindings.role") + assert.Subset(gkeSaListRoles, gkeSaRoles, fmt.Sprintf("service account %s should have project level roles", gkeServiceAgent)) + + // Cloud Armor + cloudArmorName := "eab-cloud-armor" + cloudArmorOp := gcloud.Run(t, fmt.Sprintf("compute security-policies describe %s --project %s --format json", cloudArmorName, projectID)).Array()[0] + assert.Equal(cloudArmorOp.Get("description").String(), "EAB Cloud Armor policy", "Cloud Armor description should be EAB Cloud Armor policy.") + + cluster_service_accounts := standaloneSingleProjT.GetJsonOutput("cluster_service_accounts").Array() + + assert.Greater(len(cluster_service_accounts), 0, "The terraform output must contain more than 0 service accounts.") + for _, sa := range cluster_service_accounts { + assert.True(strings.Contains(sa.String(), ".gserviceaccount.com"), "The cluster SA value must be a Google Service Account") + } + + gkeMeshCommand := fmt.Sprintf("beta container fleet mesh describe --project %s --format='json(membershipStates)'", projectID) + + membershipNamesProjectNumber := []string{} + for _, region := range clusterRegions { + membershipName := fmt.Sprintf("projects/%[1]s/locations/%[2]s/memberships/cluster-%[2]s-%[3]s", clusterProjectNumber, region, envName) + membershipNamesProjectNumber = append(membershipNamesProjectNumber, membershipName) + } + pollMeshProvisioning := func(cmd string) func() (bool, error) { + return func() (bool, error) { + retry := false + result := gcloud.Runf(t, cmd) + if len(result.Array()) < 1 { + return true, nil + } + for _, memberShipName := range membershipNamesProjectNumber { + dataPlaneManagement := result.Get("membershipStates").Get(memberShipName).Get("servicemesh.dataPlaneManagement.state").String() + controlPlaneManagement := result.Get("membershipStates").Get(memberShipName).Get("servicemesh.controlPlaneManagement.state").String() + if dataPlaneManagement == "PROVISIONING" || controlPlaneManagement == "PROVISIONING" { + retry = true + } else if !(dataPlaneManagement == "ACTIVE" && controlPlaneManagement == "ACTIVE") { + generalState := result.Get("membershipStates").Get(memberShipName).Get("state.code").String() + generalDescription := result.Get("membershipStates").Get(memberShipName).Get("state.description").String() + return false, fmt.Errorf("Service mesh provisioning failed for %s: status='%s' description='%s'", memberShipName, generalState, generalDescription) + } + } + return retry, nil + } + } + utils.Poll(t, pollMeshProvisioning(gkeMeshCommand), 40, 60*time.Second) + }) + + standaloneSingleProjT.DefineTeardown(func(assert *assert.Assertions) { + // removes firewall rules created by the service but not being deleted. + firewallRules := gcloud.Runf(t, "compute firewall-rules list --project %s --filter=\"mcsd\"", projectID).Array() + for i := range firewallRules { + gcloud.Runf(t, "compute firewall-rules delete %s --project %s -q", firewallRules[i].Get("name"), projectID) + } + standaloneSingleProjT.DefaultTeardown(assert) + + }) + // call the test function to execute the integration test + standaloneSingleProjT.Test() +} diff --git a/test/setup/README.md b/test/setup/README.md index 12032047..1f6fcd80 100644 --- a/test/setup/README.md +++ b/test/setup/README.md @@ -24,6 +24,7 @@ The Setup module creates the required prerequisite resources to deploy the bluep | envs | n/a | | org\_id | n/a | | project\_id | n/a | +| project\_id\_standalone | n/a | | sa\_key | n/a | | teams | n/a | diff --git a/test/setup/iam.tf b/test/setup/iam.tf index 10a47c81..d5c2fdd6 100644 --- a/test/setup/iam.tf +++ b/test/setup/iam.tf @@ -18,6 +18,28 @@ locals { int_required_roles = [ "roles/owner" ] + standalone_required_roles = [ + "roles/artifactregistry.admin", + "roles/cloudbuild.builds.builder", + "roles/clouddeploy.serviceAgent", + "roles/clouddeploy.admin", + "roles/compute.networkAdmin", + "roles/compute.securityAdmin", + "roles/container.admin", + "roles/gkehub.editor", + "roles/gkehub.scopeAdmin", + "roles/container.clusterAdmin", + "roles/iam.serviceAccountAdmin", + "roles/serviceusage.serviceUsageAdmin", + "roles/source.admin", + "roles/storage.admin", + "roles/resourcemanager.projectIamAdmin", + "roles/viewer", + "roles/iam.serviceAccountUser", + "roles/privilegedaccessmanager.projectServiceAgent", + "roles/logging.logWriter", + "roles/source.admin" + ] } resource "google_service_account" "int_test" { @@ -35,6 +57,14 @@ resource "google_project_iam_member" "int_test" { member = "serviceAccount:${google_service_account.int_test.email}" } +resource "google_project_iam_member" "standalone_int_test" { + for_each = toset(local.standalone_required_roles) + + project = module.project_standalone.project_id + role = each.value + member = "serviceAccount:${google_service_account.int_test.email}" +} + resource "google_organization_iam_member" "organizationServiceAgent_role" { org_id = var.org_id role = "roles/privilegedaccessmanager.organizationServiceAgent" @@ -50,3 +80,19 @@ resource "google_billing_account_iam_member" "tf_billing_user" { role = "roles/billing.admin" member = "serviceAccount:${google_service_account.int_test.email}" } + +resource "google_project_iam_member" "cb_standalone_service_agent_role" { + project = module.project_standalone.project_id + role = "roles/cloudbuild.serviceAgent" + member = "serviceAccount:service-${module.project_standalone.project_number}@gcp-sa-cloudbuild.iam.gserviceaccount.com" + + depends_on = [module.project_standalone] +} + +resource "google_project_iam_member" "cb_service_agent_role" { + project = module.project.project_id + role = "roles/cloudbuild.serviceAgent" + member = "serviceAccount:service-${module.project.project_number}@gcp-sa-cloudbuild.iam.gserviceaccount.com" + + depends_on = [module.project] +} diff --git a/test/setup/main.tf b/test/setup/main.tf index 47f0dac9..c6dbfbdf 100644 --- a/test/setup/main.tf +++ b/test/setup/main.tf @@ -68,6 +68,7 @@ module "project" { activate_apis = [ "cloudbuild.googleapis.com", + "compute.googleapis.com", "cloudresourcemanager.googleapis.com", "iam.googleapis.com", "storage-api.googleapis.com", @@ -77,13 +78,98 @@ module "project" { "sqladmin.googleapis.com", "cloudbilling.googleapis.com" ] + + activate_api_identities = [ + { + api = "compute.googleapis.com", + roles = [] + }, + { + api = "cloudbuild.googleapis.com", + roles = [ + "roles/cloudbuild.builds.builder", + "roles/cloudbuild.connectionAdmin", + ] + }, + { + api = "workflows.googleapis.com", + roles = ["roles/workflows.serviceAgent"] + }, + { + api = "config.googleapis.com", + roles = ["roles/cloudconfig.serviceAgent"] + } + ] +} + +module "project_standalone" { + source = "terraform-google-modules/project-factory/google" + version = "~> 17.0" + + name = "ci-enterprise-app-std" + random_project_id = "true" + random_project_id_length = 4 + org_id = var.org_id + folder_id = var.folder_id + billing_account = var.billing_account + deletion_policy = "DELETE" + + default_service_account = "KEEP" + + activate_api_identities = [ + { + api = "compute.googleapis.com", + roles = [] + }, + { + api = "cloudbuild.googleapis.com", + roles = [ + "roles/cloudbuild.builds.builder", + "roles/cloudbuild.connectionAdmin", + ] + }, + { + api = "workflows.googleapis.com", + roles = ["roles/workflows.serviceAgent"] + }, + { + api = "config.googleapis.com", + roles = ["roles/cloudconfig.serviceAgent"] + } + ] + + activate_apis = [ + "anthos.googleapis.com", + "anthosconfigmanagement.googleapis.com", + "apikeys.googleapis.com", + "certificatemanager.googleapis.com", + "cloudbilling.googleapis.com", + "cloudbuild.googleapis.com", + "clouddeploy.googleapis.com", + "cloudfunctions.googleapis.com", + "cloudresourcemanager.googleapis.com", + "cloudtrace.googleapis.com", + "compute.googleapis.com", + "container.googleapis.com", + "gkehub.googleapis.com", + "iam.googleapis.com", + "mesh.googleapis.com", + "multiclusteringress.googleapis.com", + "multiclusterservicediscovery.googleapis.com", + "secretmanager.googleapis.com", + "servicemanagement.googleapis.com", + "serviceusage.googleapis.com", + "sourcerepo.googleapis.com", + "sqladmin.googleapis.com", + "storage-api.googleapis.com", + "trafficdirector.googleapis.com", + ] } # Create mock common folder module "folder_common" { - source = "terraform-google-modules/folders/google" - version = "~> 5.0" - + source = "terraform-google-modules/folders/google" + version = "~> 5.0" prefix = random_string.prefix.result parent = "folders/${var.folder_id}" names = ["common"] diff --git a/test/setup/outputs.tf b/test/setup/outputs.tf index da924680..77178f34 100644 --- a/test/setup/outputs.tf +++ b/test/setup/outputs.tf @@ -18,6 +18,10 @@ output "project_id" { value = module.project.project_id } +output "project_id_standalone" { + value = module.project_standalone.project_id +} + output "sa_key" { value = google_service_account_key.int_test.private_key sensitive = true
cluster_membership_ids = list(string)
}))