[TF] More fixes for GCP DNS

* add variable to control the record TTL and lower the default from 1h
to 5min, as the rest of the records on GCP
* add variable *create_dns_records* to the testnet module as well, to
be useable from the devnet module

terraform/aptos-node-testnet/gcp/addons.tf

@@ -56,3 +56,69 @@ resource "helm_release" "chaos-mesh" {
     value = sha1(join("", [for f in fileset(local.chaos_mesh_helm_chart_path, "**") : filesha1("${local.chaos_mesh_helm_chart_path}/${f}")]))
   }
 }
+
+resource "google_service_account" "k8s-gcp-integrations" {
+  project    = var.project
+  account_id = "${local.workspace_name}-testnet-gcp"
+}
+
+resource "google_project_iam_member" "k8s-gcp-integrations-dns" {
+  project = local.zone_project
+  role    = "roles/dns.admin"
+  member  = "serviceAccount:${google_service_account.k8s-gcp-integrations.email}"
+}
+
+resource "google_service_account_iam_binding" "k8s-gcp-integrations" {
+  service_account_id = google_service_account.k8s-gcp-integrations.name
+  role               = "roles/iam.workloadIdentityUser"
+  members            = ["serviceAccount:${module.validator.gke_cluster_workload_identity_config[0].workload_pool}[kube-system/k8s-gcp-integrations]"]
+}
+
+resource "kubernetes_service_account" "k8s-gcp-integrations" {
+  metadata {
+    name      = "k8s-gcp-integrations"
+    namespace = "kube-system"
+    annotations = {
+      "iam.gke.io/gcp-service-account" = google_service_account.k8s-gcp-integrations.email
+    }
+  }
+}
+
+data "google_dns_managed_zone" "testnet" {
+  count   = var.zone_name != "" ? 1 : 0
+  name    = var.zone_name
+  project = local.zone_project
+}
+
+locals {
+  zone_project = var.zone_project != "" ? var.zone_project : var.project
+  dns_prefix   = var.workspace_dns ? "${local.workspace_name}.${var.dns_prefix_name}." : "${var.dns_prefix_name}."
+  domain       = var.zone_name != "" ? trimsuffix("${local.dns_prefix}${data.google_dns_managed_zone.testnet[0].dns_name}", ".") : null
+}
+
+resource "helm_release" "external-dns" {
+  count       = var.zone_name != "" ? 1 : 0
+  name        = "external-dns"
+  repository  = "https://kubernetes-sigs.github.io/external-dns"
+  chart       = "external-dns"
+  version     = "1.11.0"
+  namespace   = "kube-system"
+  max_history = 5
+  wait        = false
+
+  values = [
+    jsonencode({
+      serviceAccount = {
+        create = false
+        name   = kubernetes_service_account.k8s-gcp-integrations.metadata[0].name
+      }
+      provider      = "google"
+      domainFilters = var.zone_name != "" ? [data.google_dns_managed_zone.testnet[0].dns_name] : []
+      extraArgs = [
+        "--google-project=${local.zone_project}",
+        "--txt-owner-id=${local.workspace_name}",
+        "--txt-prefix=aptos",
+      ]
+    })
+  ]
+}

terraform/aptos-node-testnet/gcp/main.tf

@@ -31,7 +31,8 @@ module "validator" {
   record_name  = var.record_name
   # do not create the main fullnode and validator DNS records
   # instead, rely on external-dns from the testnet-addons
-  create_dns_records = false
+  create_dns_records = var.create_dns_records
+  dns_ttl            = var.dns_ttl
 
   # General chain config
   era            = var.era
@@ -99,6 +100,7 @@ resource "helm_release" "genesis" {
       genesis = {
         numValidators   = var.num_validators
         username_prefix = local.aptos_node_helm_prefix
+        domain          = local.domain
         validator = {
           enable_onchain_discovery = false
         }

terraform/aptos-node-testnet/gcp/variables.tf

@@ -50,6 +50,16 @@ variable "image_tag" {
 
 ### DNS config
 
+variable "workspace_dns" {
+  description = "Include Terraform workspace name in DNS records"
+  default     = true
+}
+
+variable "dns_prefix_name" {
+  description = "DNS prefix for fullnode url"
+  default     = "fullnode"
+}
+
 variable "zone_name" {
   description = "Zone name of GCP Cloud DNS zone to create records in"
   default     = ""
@@ -65,6 +75,16 @@ variable "record_name" {
   default     = "<workspace>.aptos"
 }
 
+variable "create_dns_records" {
+  description = "Creates DNS records in var.zone_name that point to k8s service, as opposed to using external-dns or other means"
+  default     = true
+}
+
+variable "dns_ttl" {
+  description = "Time-to-Live for the Validator and Fullnode DNS records"
+  default     = 300
+}
+
 ### Testnet config
 
 variable "workspace_name_override" {

terraform/aptos-node/gcp/cluster.tf

@@ -13,6 +13,10 @@ resource "google_container_cluster" "aptos" {
     channel = "REGULAR"
   }
 
+  pod_security_policy_config {
+    enabled = false
+  }
+
   master_auth {
     client_certificate_config {
       issue_client_certificate = false

terraform/aptos-node/gcp/dns.tf

@@ -10,13 +10,14 @@ resource "random_string" "validator-dns" {
 }
 
 locals {
+  dns_prefix  = var.workspace_dns ? "${local.workspace_name}." : ""
   record_name = replace(var.record_name, "<workspace>", local.workspace_name)
+  domain      = var.zone_name != "" ? "${local.dns_prefix}${data.google_dns_managed_zone.aptos[0].dns_name}" : null
 }
 
 data "kubernetes_service" "validator-lb" {
   count = var.zone_name != "" && var.create_dns_records ? 1 : 0
   metadata {
-    # This is the main validator LB service that is created by the aptos-node helm chart
     name = "${local.workspace_name}-aptos-node-0-validator-lb"
   }
   depends_on = [time_sleep.lb_creation]
@@ -25,7 +26,6 @@ data "kubernetes_service" "validator-lb" {
 data "kubernetes_service" "fullnode-lb" {
   count = var.zone_name != "" && var.create_dns_records ? 1 : 0
   metadata {
-    # This is the main fullnode LB service that is created by the aptos-node helm chart
     name = "${local.workspace_name}-aptos-node-0-fullnode-lb"
   }
   depends_on = [time_sleep.lb_creation]
@@ -43,7 +43,7 @@ resource "google_dns_record_set" "validator" {
   project      = data.google_dns_managed_zone.aptos[0].project
   name         = "${random_string.validator-dns.result}.${local.record_name}.${data.google_dns_managed_zone.aptos[0].dns_name}"
   type         = "A"
-  ttl          = 3600
+  ttl          = var.dns_ttl
   rrdatas      = [data.kubernetes_service.validator-lb[0].status[0].load_balancer[0].ingress[0].ip]
 }
 
@@ -53,7 +53,7 @@ resource "google_dns_record_set" "fullnode" {
   project      = data.google_dns_managed_zone.aptos[0].project
   name         = "${local.record_name}.${data.google_dns_managed_zone.aptos[0].dns_name}"
   type         = "A"
-  ttl          = 3600
+  ttl          = var.dns_ttl
   rrdatas      = [data.kubernetes_service.fullnode-lb[0].status[0].load_balancer[0].ingress[0].ip]
 }
 

terraform/aptos-node/gcp/kubernetes.tf

@@ -77,6 +77,14 @@ resource "helm_release" "validator" {
           effect = "NoExecute"
         }]
       }
+      haproxy = {
+        nodeSelector = var.gke_enable_node_autoprovisioning ? {} : {
+          "cloud.google.com/gke-nodepool" = google_container_node_pool.utilities.name
+        }
+      }
+      service = {
+        domain = local.domain
+      }
     }),
     var.helm_values_file != "" ? file(var.helm_values_file) : "{}",
     jsonencode(var.helm_values),

terraform/aptos-node/gcp/outputs.tf

@@ -9,3 +9,7 @@ output "gke_cluster_endpoint" {
 output "gke_cluster_ca_certificate" {
   value = google_container_cluster.aptos.master_auth[0].cluster_ca_certificate
 }
+
+output "gke_cluster_workload_identity_config" {
+  value = google_container_cluster.aptos.workload_identity_config
+}

terraform/aptos-node/gcp/security.tf

@@ -1,44 +1,20 @@
 # Security-related resources
 
-data "kubernetes_all_namespaces" "all" {
-  count = var.cluster_bootstrap ? 0 : 1
-}
-
 locals {
-  kubernetes_master_version = substr(google_container_cluster.aptos.master_version, 0, 4)
-  baseline_pss_labels = {
+  # Enforce "privileged" PSS (i.e. allow everything), but warn about
+  # infractions of "baseline" profile
+  privileged_pss_labels = {
     "pod-security.kubernetes.io/audit"   = "baseline"
     "pod-security.kubernetes.io/warn"    = "baseline"
     "pod-security.kubernetes.io/enforce" = "privileged"
   }
 }
 
-# FIXME: Remove after migration to K8s 1.25
-resource "kubernetes_role_binding" "disable-psp" {
-  for_each = toset(var.cluster_bootstrap ? [] : local.kubernetes_master_version <= "1.24" ? data.kubernetes_all_namespaces.all[0].namespaces : [])
-  metadata {
-    name      = "privileged-psp"
-    namespace = each.value
-  }
-
-  role_ref {
-    api_group = "rbac.authorization.k8s.io"
-    kind      = "ClusterRole"
-    name      = "gce:podsecuritypolicy:privileged"
-  }
-
-  subject {
-    api_group = "rbac.authorization.k8s.io"
-    kind      = "Group"
-    name      = "system:serviceaccounts:${each.value}"
-  }
-}
-
 resource "kubernetes_labels" "pss-default" {
   api_version = "v1"
   kind        = "Namespace"
   metadata {
     name = "default"
   }
-  labels = local.baseline_pss_labels
+  labels = local.privileged_pss_labels
 }

terraform/aptos-node/gcp/variables.tf

@@ -46,26 +46,6 @@ variable "image_tag" {
   default     = "devnet"
 }
 
-variable "zone_name" {
-  description = "Zone name of GCP Cloud DNS zone to create records in"
-  default     = ""
-}
-
-variable "zone_project" {
-  description = "GCP project which the DNS zone is in (if different)"
-  default     = ""
-}
-
-variable "record_name" {
-  description = "DNS record name to use (<workspace> is replaced with the TF workspace name)"
-  default     = "<workspace>.aptos"
-}
-
-variable "create_dns_records" {
-  description = "Creates DNS records in var.zone_name that point to k8s service, as opposed to using external-dns or other means"
-  default     = true
-}
-
 variable "helm_chart" {
   description = "Path to aptos-validator Helm chart file"
   default     = ""
@@ -171,8 +151,39 @@ variable "manage_via_tf" {
   default     = true
 }
 
-### Autoscaling
+### DNS
 
+variable "zone_name" {
+  description = "Zone name of GCP Cloud DNS zone to create records in"
+  default     = ""
+}
+
+variable "zone_project" {
+  description = "GCP project which the DNS zone is in (if different)"
+  default     = ""
+}
+
+variable "workspace_dns" {
+  description = "Include Terraform workspace name in DNS records"
+  default     = true
+}
+
+variable "record_name" {
+  description = "DNS record name to use (<workspace> is replaced with the TF workspace name)"
+  default     = "<workspace>.aptos"
+}
+
+variable "create_dns_records" {
+  description = "Creates DNS records in var.zone_name that point to k8s service, as opposed to using external-dns or other means"
+  default     = true
+}
+
+variable "dns_ttl" {
+  description = "Time-to-Live for the Validator and Fullnode DNS records"
+  default     = 300
+}
+
+### Autoscaling
 
 variable "gke_enable_node_autoprovisioning" {
   description = "Enable node autoprovisioning for GKE cluster. See https://cloud.google.com/kubernetes-engine/docs/how-to/node-auto-provisioning"

terraform/aptos-node/gcp/versions.tf

@@ -2,10 +2,12 @@ terraform {
   required_version = "~> 1.3.6"
   required_providers {
     google = {
-      source = "hashicorp/google"
+      source  = "hashicorp/google"
+      version = "~> 4.54.0"
     }
     google-beta = {
-      source = "hashicorp/google-beta"
+      source  = "hashicorp/google-beta"
+      version = "~> 4.54.0"
     }
     helm = {
       source = "hashicorp/helm"