feat(acm): remove direct kubectl commands

kubectl-wrapper currently breaks if one has to access the api using a
proxy (IAP).

modules/acm/README.md

@@ -95,6 +95,7 @@ data "google_client_config" "default" {}
 | policy\_bundles | A list of Policy Controller policy bundles git urls (example: https://github.com/GoogleCloudPlatform/acm-policy-controller-library.git/bundles/policy-essentials-v2022) to install on the cluster. | `list(string)` | `[]` | no |
 | policy\_dir | Subfolder containing configs in ACM Git repo. If un-set, uses Config Management default. | `string` | `""` | no |
 | project\_id | GCP project\_id used to reach cluster. | `string` | n/a | yes |
+| restart\_gatekeeper\_controller\_manager | Restart the gatekeeper controller manager after setting up workload id (needs to be done manually if a proxy to gke api is required) | `bool` | `true` | no |
 | secret\_type | git authentication secret type, is passed through to ConfigManagement spec.git.secretType. Overriden to value 'ssh' if `create_ssh_key` is true | `string` | `"ssh"` | no |
 | source\_format | Configures a non-hierarchical repo if set to 'unstructured'. Uses [ACM defaults](https://cloud.google.com/anthos-config-management/docs/how-to/installing#configuring-config-management-operator) when unset. | `string` | `""` | no |
 | ssh\_auth\_key | Key for Git authentication. Overrides 'create\_ssh\_key' variable. Can be set using 'file(path/to/file)'-function. | `string` | `null` | no |

modules/acm/creds.tf

@@ -39,54 +39,57 @@ resource "time_sleep" "wait_acm" {
 }
 
 resource "google_service_account_iam_binding" "ksa_iam" {
-  count              = length(local.iam_ksa_binding_members) > 0 ? 1 : 0
+  count      = length(local.iam_ksa_binding_members) > 0 ? 1 : 0
+  depends_on = [google_gke_hub_feature_membership.main]
+
   service_account_id = google_service_account.acm_metrics_writer_sa[0].name
   role               = "roles/iam.workloadIdentityUser"
 
   members = [
     for ksa in local.iam_ksa_binding_members : "serviceAccount:${var.project_id}.svc.id.goog[${ksa}]"
   ]
-
-  depends_on = [google_gke_hub_feature_membership.main]
 }
 
-module "annotate-sa-config-management-monitoring" {
-  source  = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper"
-  version = "~> 3.1"
+resource "kubernetes_annotations" "annotate-sa-config-management-monitoring" {
+  count = var.enable_config_sync && var.create_metrics_gcp_sa ? 1 : 0
 
-  count            = var.enable_config_sync && var.create_metrics_gcp_sa ? 1 : 0
-  skip_download    = true
-  cluster_name     = var.cluster_name
-  cluster_location = var.location
-  project_id       = var.project_id
+  api_version = "v1"
+  kind        = "ServiceAccount"
 
-  kubectl_create_command  = "kubectl annotate --overwrite sa -n config-management-monitoring default iam.gke.io/gcp-service-account=${google_service_account.acm_metrics_writer_sa[0].email}"
-  kubectl_destroy_command = "kubectl annotate sa -n config-management-monitoring default iam.gke.io/gcp-service-account-"
+  metadata {
+    name      = "default"
+    namespace = "config-management-monitoring"
+  }
 
-  module_depends_on = time_sleep.wait_acm
+  annotations = {
+    "iam.gke.io/gcp-service-account" : google_service_account.acm_metrics_writer_sa[0].email
+  }
+
+  depends_on = [time_sleep.wait_acm]
 }
 
-module "annotate-sa-gatekeeper-system" {
-  source  = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper"
-  version = "~> 3.1"
+resource "kubernetes_annotations" "annotate-sa-gatekeeper-system" {
+  count      = var.enable_policy_controller && var.create_metrics_gcp_sa ? 1 : 0
+  depends_on = [time_sleep.wait_acm]
 
-  count            = var.enable_policy_controller && var.create_metrics_gcp_sa ? 1 : 0
-  skip_download    = true
-  cluster_name     = var.cluster_name
-  cluster_location = var.location
-  project_id       = var.project_id
+  api_version = "v1"
+  kind        = "ServiceAccount"
 
-  kubectl_create_command  = "kubectl annotate --overwrite sa -n gatekeeper-system gatekeeper-admin iam.gke.io/gcp-service-account=${google_service_account.acm_metrics_writer_sa[0].email}"
-  kubectl_destroy_command = "kubectl annotate sa -n gatekeeper-system gatekeeper-admin iam.gke.io/gcp-service-account-"
+  metadata {
+    name      = "gatekeeper-admin"
+    namespace = "gatekeeper-system"
+  }
 
-  module_depends_on = time_sleep.wait_acm
+  annotations = {
+    "iam.gke.io/gcp-service-account" : google_service_account.acm_metrics_writer_sa[0].email
+  }
 }
 
 module "annotate-sa-gatekeeper-system-restart" {
   source  = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper"
   version = "~> 3.1"
 
-  count            = var.enable_policy_controller && var.create_metrics_gcp_sa ? 1 : 0
+  count            = var.enable_policy_controller && var.create_metrics_gcp_sa && var.restart_gatekeeper_controller_manager ? 1 : 0
   skip_download    = true
   cluster_name     = var.cluster_name
   cluster_location = var.location
@@ -95,7 +98,7 @@ module "annotate-sa-gatekeeper-system-restart" {
   kubectl_create_command  = "kubectl rollout restart deployment gatekeeper-controller-manager -n gatekeeper-system"
   kubectl_destroy_command = ""
 
-  module_depends_on = module.annotate-sa-gatekeeper-system
+  module_depends_on = resource.kubernetes_annotations.annotate-sa-gatekeeper-system
 }
 
 resource "google_service_account" "acm_metrics_writer_sa" {

modules/acm/variables.tf

@@ -176,3 +176,9 @@ variable "metrics_gcp_sa_name" {
   type        = string
   default     = "acm-metrics-writer"
 }
+
+variable "restart_gatekeeper_controller_manager" {
+  description = "Restart the gatekeeper controller manager after setting up workload id (needs to be done manually if a proxy to gke api is required)"
+  type        = bool
+  default     = true
+}