Added support for multi-project GKE Hub registration

Added an optional variable HUB_PROJECT_ID which will allow you to specific a seperate GCP project for the GKE HUB than the project the cluster is deployed to. This included updating the 3 examples the leveraged the hub module. Issue: terraform-google-modules#837
JamesDuncanNz · Mar 3, 2021 · a7807a2 · a7807a2
1 parent 3354205
commit a7807a2
Show file tree

Hide file tree

Showing 14 changed files with 63 additions and 14 deletions.
diff --git a/examples/simple_zonal_with_asm/README.md b/examples/simple_zonal_with_asm/README.md
@@ -8,6 +8,7 @@ This example illustrates how to create a simple zonal cluster with ASM.
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | cluster\_name\_suffix | A suffix to append to the default cluster name | `string` | `""` | no |
+| hub\_project\_id | The project in which the GKE Hub belongs. | `string` | `""` | no |
 | ip\_range\_pods | The secondary ip range to use for pods | `any` | n/a | yes |
 | ip\_range\_services | The secondary ip range to use for services | `any` | n/a | yes |
 | network | The VPC network to host the cluster in | `any` | n/a | yes |

diff --git a/examples/simple_zonal_with_asm/hub.tf b/examples/simple_zonal_with_asm/hub.tf
@@ -17,6 +17,7 @@
 module "hub" {
   source                  = "../../modules/hub"
   project_id              = var.project_id
+  hub_project_id          = var.hub_project_id
   location                = module.gke.location
   cluster_name            = module.gke.name
   cluster_endpoint        = module.gke.endpoint

diff --git a/examples/simple_zonal_with_asm/variables.tf b/examples/simple_zonal_with_asm/variables.tf
@@ -18,6 +18,11 @@ variable "project_id" {
   description = "The project ID to host the cluster in"
 }
 
+variable "hub_project_id" {
+  description = "The project in which the GKE Hub belongs."
+  default     = ""
+}
+
 variable "cluster_name_suffix" {
   description = "A suffix to append to the default cluster name"
   default     = ""

diff --git a/examples/simple_zonal_with_hub/README.md b/examples/simple_zonal_with_hub/README.md
@@ -10,6 +10,7 @@ It incorporates the standard cluster module and the [Hub registration module](..
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | cluster\_name\_suffix | A suffix to append to the default cluster name | `string` | `""` | no |
+| hub\_project\_id | The project in which the GKE Hub belongs. | `string` | `""` | no |
 | ip\_range\_pods | The secondary ip range to use for pods | `string` | `""` | no |
 | ip\_range\_services | The secondary ip range to use for services | `string` | `""` | no |
 | network | The VPC network to host the cluster in | `string` | `"default"` | no |

diff --git a/examples/simple_zonal_with_hub/hub.tf b/examples/simple_zonal_with_hub/hub.tf
@@ -17,6 +17,7 @@
 module "hub" {
   source           = "../../modules/hub"
   project_id       = var.project_id
+  hub_project_id   = var.hub_project_id
   location         = module.gke.location
   cluster_name     = module.gke.name
   cluster_endpoint = module.gke.endpoint

diff --git a/examples/simple_zonal_with_hub/variables.tf b/examples/simple_zonal_with_hub/variables.tf
@@ -18,6 +18,11 @@ variable "project_id" {
   description = "The project ID to host the cluster in"
 }
 
+variable "hub_project_id" {
+  description = "The project in which the GKE Hub belongs."
+  default     = ""
+}
+
 variable "cluster_name_suffix" {
   description = "A suffix to append to the default cluster name"
   default     = ""

diff --git a/examples/simple_zonal_with_hub_kubeconfig/README.md b/examples/simple_zonal_with_hub_kubeconfig/README.md
@@ -9,7 +9,8 @@ It creates a [kind](https://kind.sigs.k8s.io/) cluster, sets current kubecontext
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| project\_id | The project ID (environ) to register the cluster in | `any` | n/a | yes |
+| hub\_project\_id | The project in which the GKE Hub belongs. | `string` | `""` | no |
+| project\_id | The project ID to host the cluster in | `any` | n/a | yes |
 
 ## Outputs
 

diff --git a/examples/simple_zonal_with_hub_kubeconfig/hub.tf b/examples/simple_zonal_with_hub_kubeconfig/hub.tf
@@ -17,6 +17,7 @@
 module "hub" {
   source                  = "../../modules/hub"
   project_id              = var.project_id
+  hub_project_id          = var.hub_project_id
   location                = "remote"
   cluster_name            = kind_cluster.test-cluster.name
   cluster_endpoint        = kind_cluster.test-cluster.endpoint

diff --git a/examples/simple_zonal_with_hub_kubeconfig/variables.tf b/examples/simple_zonal_with_hub_kubeconfig/variables.tf
@@ -15,5 +15,10 @@
  */
 
 variable "project_id" {
-  description = "The project ID (environ) to register the cluster in"
+  description = "The project ID to host the cluster in"
+}
+
+variable "hub_project_id" {
+  description = "The project in which the GKE Hub belongs."
+  default     = ""
 }
diff --git a/modules/hub/README.md b/modules/hub/README.md
@@ -39,6 +39,7 @@ To deploy this config:
 | gcloud\_sdk\_version | The gcloud sdk version to use. Minimum required version is 293.0.0 | `string` | `"296.0.1"` | no |
 | gke\_hub\_membership\_name | Membership name that uniquely represents the cluster being registered on the Hub | `string` | `"gke-hub-membership"` | no |
 | gke\_hub\_sa\_name | Name for the GKE Hub SA stored as a secret `creds-gcp` in the `gke-connect` namespace. | `string` | `"gke-hub-sa"` | no |
+| hub\_project\_id | The project in which the GKE Hub belongs. | `string` | `""` | no |
 | labels | Comma separated labels in the format name=value to apply to cluster in the GCP Console. | `string` | `""` | no |
 | location | The location (zone or region) this cluster has been created in. | `string` | n/a | yes |
 | module\_depends\_on | List of modules or resources this module depends on. | `list` | `[]` | no |

diff --git a/modules/hub/main.tf b/modules/hub/main.tf
@@ -19,28 +19,45 @@ locals {
 
   is_gke_flag               = var.use_kubeconfig ? 0 : 1
   create_cmd_gke_entrypoint = "${path.module}/scripts/gke_hub_registration.sh"
-  create_cmd_gke_body       = "${local.is_gke_flag} ${var.gke_hub_membership_name} ${var.location} ${var.cluster_name} ${local.gke_hub_sa_key} ${var.project_id} ${var.labels}"
+  create_cmd_gke_body       = "${local.is_gke_flag} ${var.gke_hub_membership_name} ${var.location} ${var.cluster_name} ${local.gke_hub_sa_key} ${var.project_id} ${var.hub_project_id == "" ? var.project_id : var.hub_project_id} ${var.labels}"
   destroy_gke_entrypoint    = "${path.module}/scripts/gke_hub_unregister.sh"
-  destroy_gke_body          = "${local.is_gke_flag} ${var.gke_hub_membership_name} ${var.location} ${var.cluster_name} ${var.project_id}"
+  destroy_gke_body          = "${local.is_gke_flag} ${var.gke_hub_membership_name} ${var.location} ${var.cluster_name} ${var.project_id} ${var.hub_project_id == "" ? var.project_id : var.hub_project_id}"
 }
 
 data "google_client_config" "default" {
 }
 
+data "google_project" "project" {
+  project_id = var.hub_project_id == "" ? var.project_id : var.hub_project_id
+}
+
 resource "google_service_account" "gke_hub_sa" {
   count        = var.use_existing_sa ? 0 : 1
   account_id   = var.gke_hub_sa_name
-  project      = var.project_id
+  project      = var.hub_project_id == "" ? var.project_id : var.hub_project_id
   display_name = "Service Account for GKE Hub Registration"
 }
 
 resource "google_project_iam_member" "gke_hub_member" {
   count   = var.use_existing_sa ? 0 : 1
-  project = var.project_id
+  project = var.hub_project_id == "" ? var.project_id : var.hub_project_id
   role    = "roles/gkehub.connect"
   member  = "serviceAccount:${google_service_account.gke_hub_sa[0].email}"
 }
 
+resource "google_project_iam_member" "hub_service_agent_project" {
+  project = var.project_id
+  role    = "roles/gkehub.serviceAgent"
+  member  = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-gkehub.iam.gserviceaccount.com"
+}
+
+resource "google_project_iam_member" "hub_service_agent_hub" {
+  count   = var.hub_project_id == "" ? 0 : 1
+  project = var.hub_project_id
+  role    = "roles/gkehub.serviceAgent"
+  member  = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-gkehub.iam.gserviceaccount.com"
+}
+
 resource "google_service_account_key" "gke_hub_key" {
   count              = var.use_existing_sa ? 0 : 1
   service_account_id = google_service_account.gke_hub_sa[0].name

diff --git a/modules/hub/scripts/gke_hub_registration.sh b/modules/hub/scripts/gke_hub_registration.sh
@@ -15,7 +15,7 @@
 
 set -e
 
-if [ "$#" -lt 5 ]; then
+if [ "$#" -lt 6 ]; then
     >&2 echo "Not all expected arguments set."
     exit 1
 fi
@@ -26,7 +26,8 @@ CLUSTER_LOCATION=$3
 CLUSTER_NAME=$4
 SERVICE_ACCOUNT_KEY=$5
 PROJECT_ID=$6
-LABELS=$7
+HUB_PROJECT_ID=$7
+LABELS=$8
 
 #write temp key, cleanup at exit
 tmp_file=$(mktemp)
@@ -37,18 +38,19 @@ echo "${SERVICE_ACCOUNT_KEY}" | base64 ${B64_ARG} > "$tmp_file"
 
 if [[ ${GKE_CLUSTER_FLAG} == 1 ]]; then
     echo "Registering GKE Cluster."
-    gcloud container hub memberships register "${MEMBERSHIP_NAME}" --gke-cluster="${CLUSTER_LOCATION}"/"${CLUSTER_NAME}" --service-account-key-file="${tmp_file}" --project="${PROJECT_ID}" --quiet
+    CLUSTER_URI="https://container.googleapis.com/projects/${PROJECT_ID}/locations/${CLUSTER_LOCATION}/clusters/${CLUSTER_NAME}"
+    gcloud container hub memberships register "${MEMBERSHIP_NAME}" --gke-uri="${CLUSTER_URI}" --service-account-key-file="${tmp_file}" --project="${HUB_PROJECT_ID}" --quiet
 else
     echo "Registering a non-GKE Cluster. Using current-context to register Hub membership."
     #Get the kubeconfig
     CONTEXT=$(kubectl config current-context)
-    gcloud container hub memberships register "${MEMBERSHIP_NAME}" --context="${CONTEXT}" --service-account-key-file="${tmp_file}" --project="${PROJECT_ID}" --quiet
+    gcloud container hub memberships register "${MEMBERSHIP_NAME}" --context="${CONTEXT}" --service-account-key-file="${tmp_file}" --project="${HUB_PROJECT_ID}" --quiet
 fi
 
 
 # Add labels to the registered cluster
 if [ -z ${LABELS+x} ]; then
     echo "No hub labels to apply."
 else
-    gcloud container hub memberships update "${MEMBERSHIP_NAME}" --update-labels "$LABELS" --project="${PROJECT_ID}"
+    gcloud container hub memberships update "${MEMBERSHIP_NAME}" --update-labels "$LABELS" --project="${HUB_PROJECT_ID}"
 fi
diff --git a/modules/hub/scripts/gke_hub_unregister.sh b/modules/hub/scripts/gke_hub_unregister.sh
@@ -15,7 +15,7 @@
 
 set -e
 
-if [ "$#" -lt 5 ]; then
+if [ "$#" -lt 6 ]; then
     >&2 echo "Not all expected arguments set."
     exit 1
 fi
@@ -25,15 +25,17 @@ MEMBERSHIP_NAME=$2
 CLUSTER_LOCATION=$3
 CLUSTER_NAME=$4
 PROJECT_ID=$5
+HUB_PROJECT_ID=$6
 
 
 
 if [[ ${GKE_CLUSTER_FLAG} == 1 ]]; then
     echo "Un-Registering GKE Cluster."
-    gcloud container hub memberships unregister "${MEMBERSHIP_NAME}" --gke-cluster="${CLUSTER_LOCATION}"/"${CLUSTER_NAME}" --project "${PROJECT_ID}"
+    CLUSTER_URI="https://container.googleapis.com/projects/${PROJECT_ID}/locations/${CLUSTER_LOCATION}/clusters/${CLUSTER_NAME}"
+    gcloud container hub memberships unregister "${MEMBERSHIP_NAME}" --gke-uri="${CLUSTER_URI}" --project "${HUB_PROJECT_ID}"
 else
     echo "Un-Registering a non-GKE Cluster. Using current-context to unregister Hub membership."
     #Get Current context
     CONTEXT=$(kubectl config current-context)
-    gcloud container hub memberships unregister "${MEMBERSHIP_NAME}" --context="${CONTEXT}" --project="${PROJECT_ID}"
+    gcloud container hub memberships unregister "${MEMBERSHIP_NAME}" --context="${CONTEXT}" --project="${HUB_PROJECT_ID}"
 fi
diff --git a/modules/hub/variables.tf b/modules/hub/variables.tf
@@ -29,6 +29,12 @@ variable "project_id" {
   type        = string
 }
 
+variable "hub_project_id" {
+  description = "The project in which the GKE Hub belongs."
+  type        = string
+  default     = ""
+}
+
 variable "location" {
   description = "The location (zone or region) this cluster has been created in."
   type        = string