From 54eca6b6469517495711c54dab3413003a58a410 Mon Sep 17 00:00:00 2001
From: gcamus59 <45559998+gcamus59@users.noreply.github.com>
Date: Thu, 23 Apr 2020 17:15:35 +0200
Subject: [PATCH] feat: Add fully configurable resource usage export block in
 GA and upgrade GCP provider (#491)

BREAKING CHANGE: Minimum Google provider version increased to 3.16.
---
 README.md                                     |  3 +++
 autogen/main/cluster.tf.tmpl                  | 26 ++++++++++++-------
 autogen/main/variables.tf.tmpl                | 24 ++++++++++++-----
 autogen/main/versions.tf.tmpl                 |  4 +--
 autogen/safer-cluster/variables.tf.tmpl       | 12 +++++++++
 cluster.tf                                    | 16 ++++++++++++
 examples/deploy_service/main.tf               |  2 +-
 examples/disable_client_cert/main.tf          |  2 +-
 examples/node_pool/main.tf                    |  2 +-
 examples/node_pool_update_variant/main.tf     |  2 +-
 .../node_pool_update_variant_beta/main.tf     |  2 +-
 .../provider.tf                               |  4 +--
 examples/safer_cluster/main.tf                |  4 +--
 examples/shared_vpc/main.tf                   |  2 +-
 examples/simple_regional/main.tf              |  2 +-
 examples/simple_regional_beta/main.tf         |  2 +-
 examples/simple_regional_private/main.tf      |  2 +-
 examples/simple_regional_private_beta/main.tf |  4 +--
 .../simple_regional_with_kubeconfig/main.tf   |  2 +-
 .../simple_regional_with_networking/main.tf   |  2 +-
 examples/simple_zonal_private/main.tf         |  2 +-
 examples/simple_zonal_with_acm/main.tf        |  2 +-
 examples/stub_domains/main.tf                 |  2 +-
 examples/stub_domains_private/main.tf         |  2 +-
 .../stub_domains_upstream_nameservers/main.tf |  2 +-
 examples/upstream_nameservers/main.tf         |  2 +-
 examples/workload_identity/main.tf            |  2 +-
 examples/workload_metadata_config/main.tf     |  2 +-
 .../README.md                                 |  4 ++-
 .../cluster.tf                                | 26 ++++++++++++-------
 .../variables.tf                              | 24 ++++++++++++-----
 .../versions.tf                               |  2 +-
 modules/beta-private-cluster/README.md        |  4 ++-
 modules/beta-private-cluster/cluster.tf       | 26 ++++++++++++-------
 modules/beta-private-cluster/variables.tf     | 24 ++++++++++++-----
 modules/beta-private-cluster/versions.tf      |  2 +-
 modules/beta-public-cluster/README.md         |  4 ++-
 modules/beta-public-cluster/cluster.tf        | 26 ++++++++++++-------
 modules/beta-public-cluster/variables.tf      | 24 ++++++++++++-----
 modules/beta-public-cluster/versions.tf       |  2 +-
 .../private-cluster-update-variant/README.md  |  3 +++
 .../private-cluster-update-variant/cluster.tf | 16 ++++++++++++
 .../variables.tf                              | 18 +++++++++++++
 .../versions.tf                               |  2 +-
 modules/private-cluster/README.md             |  3 +++
 modules/private-cluster/cluster.tf            | 16 ++++++++++++
 modules/private-cluster/variables.tf          | 18 +++++++++++++
 modules/private-cluster/versions.tf           |  2 +-
 .../safer-cluster-update-variant/README.md    |  2 ++
 .../safer-cluster-update-variant/variables.tf | 12 +++++++++
 modules/safer-cluster/README.md               |  2 ++
 modules/safer-cluster/variables.tf            | 12 +++++++++
 test/fixtures/deploy_service/network.tf       |  2 +-
 test/fixtures/disable_client_cert/network.tf  |  2 +-
 test/fixtures/shared_vpc/network.tf           |  2 +-
 test/fixtures/simple_regional/network.tf      |  2 +-
 .../network.tf                                |  2 +-
 test/fixtures/simple_zonal/network.tf         |  2 +-
 test/fixtures/stub_domains/network.tf         |  2 +-
 .../network.tf                                |  2 +-
 test/fixtures/upstream_nameservers/network.tf |  2 +-
 variables.tf                                  | 18 +++++++++++++
 versions.tf                                   |  2 +-
 63 files changed, 338 insertions(+), 109 deletions(-)

diff --git a/README.md b/README.md
index b4ebb2cf7..51d1ff999 100644
--- a/README.md
+++ b/README.md
@@ -110,6 +110,8 @@ Then perform the following commands on the root folder:
 | default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no |
 | description | The description of the cluster | string | `""` | no |
 | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no |
+| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | bool | `"false"` | no |
+| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | bool | `"true"` | no |
 | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers | list(string) | `<list>` | no |
 | firewall\_priority | Priority rule for firewall rules | number | `"1000"` | no |
 | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
@@ -142,6 +144,7 @@ Then perform the following commands on the root folder:
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
 | registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
+| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | string | `""` | no |
 | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
 | skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no |
 | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `<map>` | no |
diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl
index 841b8cf08..920470f06 100644
--- a/autogen/main/cluster.tf.tmpl
+++ b/autogen/main/cluster.tf.tmpl
@@ -99,16 +99,6 @@ resource "google_container_cluster" "primary" {
       enabled = pod_security_policy_config.value.enabled
     }
   }
-
-  dynamic "resource_usage_export_config" {
-    for_each = var.resource_usage_export_dataset_id != "" ? [var.resource_usage_export_dataset_id] : []
-    content {
-      enable_network_egress_metering = true
-      bigquery_destination {
-        dataset_id = resource_usage_export_config.value
-      }
-    }
-  }
 {% endif %}
   dynamic "master_authorized_networks_config" {
     for_each = local.master_authorized_networks_config
@@ -223,6 +213,22 @@ resource "google_container_cluster" "primary" {
     }
   }
 
+  dynamic "resource_usage_export_config" {
+    for_each = var.resource_usage_export_dataset_id != "" ? [{
+      enable_network_egress_metering       = var.enable_network_egress_export
+      enable_resource_consumption_metering = var.enable_resource_consumption_export
+      dataset_id                           = var.resource_usage_export_dataset_id
+    }] : []
+
+    content {
+      enable_network_egress_metering       = resource_usage_export_config.value.enable_network_egress_metering
+      enable_resource_consumption_metering = resource_usage_export_config.value.enable_resource_consumption_metering
+      bigquery_destination {
+        dataset_id = resource_usage_export_config.value.dataset_id
+      }
+    }
+  }
+
 {% if private_cluster %}
   dynamic "private_cluster_config" {
     for_each = var.enable_private_nodes ? [{
diff --git a/autogen/main/variables.tf.tmpl b/autogen/main/variables.tf.tmpl
index 4525819e5..75f18de9b 100644
--- a/autogen/main/variables.tf.tmpl
+++ b/autogen/main/variables.tf.tmpl
@@ -182,6 +182,24 @@ variable "node_pools_metadata" {
     default-node-pool = {}
   }
 }
+
+variable "resource_usage_export_dataset_id" {
+  type        = string
+  description = "The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export."
+  default     = ""
+}
+
+variable "enable_network_egress_export" {
+  type        = bool
+  description = "Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic."
+  default     = false
+}
+
+variable "enable_resource_consumption_export" {
+  type        = bool
+  description = "Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export."
+  default     = true
+}
 {% if beta_cluster %}
 
 variable "enable_kubernetes_alpha" {
@@ -428,12 +446,6 @@ variable "pod_security_policy_config" {
   }]
 }
 
-variable "resource_usage_export_dataset_id" {
-  type        = string
-  description = "The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic."
-  default     = ""
-}
-
 variable "node_metadata" {
   description = "Specifies how node metadata is exposed to the workload running on the node"
   default     = "SECURE"
diff --git a/autogen/main/versions.tf.tmpl b/autogen/main/versions.tf.tmpl
index 79e8226f0..b7c1232b2 100644
--- a/autogen/main/versions.tf.tmpl
+++ b/autogen/main/versions.tf.tmpl
@@ -19,9 +19,9 @@ terraform {
 
   required_providers {
 {% if beta_cluster %}
-    google-beta = ">= 3.1, <4.0.0"
+    google-beta = ">= 3.16, <4.0.0"
 {% else %}
-    google = ">= 2.18, <4.0.0"
+    google = ">= 3.16, <4.0.0"
 {% endif %}
   }
 }
diff --git a/autogen/safer-cluster/variables.tf.tmpl b/autogen/safer-cluster/variables.tf.tmpl
index 1d82c13b9..85d9555a1 100644
--- a/autogen/safer-cluster/variables.tf.tmpl
+++ b/autogen/safer-cluster/variables.tf.tmpl
@@ -268,6 +268,18 @@ variable "resource_usage_export_dataset_id" {
   default     = ""
 }
 
+variable "enable_network_egress_export" {
+  type        = bool
+  description = "Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic."
+  default     = false
+}
+
+variable "enable_resource_consumption_export" {
+  type        = bool
+  description = "Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export."
+  default     = true
+}
+
 variable "sandbox_enabled" {
   type        = bool
   description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it)."
diff --git a/cluster.tf b/cluster.tf
index b9c937e18..d8781d40f 100644
--- a/cluster.tf
+++ b/cluster.tf
@@ -118,6 +118,22 @@ resource "google_container_cluster" "primary" {
     }
   }
 
+  dynamic "resource_usage_export_config" {
+    for_each = var.resource_usage_export_dataset_id != "" ? [{
+      enable_network_egress_metering       = var.enable_network_egress_export
+      enable_resource_consumption_metering = var.enable_resource_consumption_export
+      dataset_id                           = var.resource_usage_export_dataset_id
+    }] : []
+
+    content {
+      enable_network_egress_metering       = resource_usage_export_config.value.enable_network_egress_metering
+      enable_resource_consumption_metering = resource_usage_export_config.value.enable_resource_consumption_metering
+      bigquery_destination {
+        dataset_id = resource_usage_export_config.value.dataset_id
+      }
+    }
+  }
+
 
   remove_default_node_pool = var.remove_default_node_pool
 }
diff --git a/examples/deploy_service/main.tf b/examples/deploy_service/main.tf
index 0a179e413..341604ae2 100644
--- a/examples/deploy_service/main.tf
+++ b/examples/deploy_service/main.tf
@@ -19,7 +19,7 @@ locals {
 }
 
 provider "google" {
-  version = "~> 3.14.0"
+  version = "~> 3.16.0"
   region  = var.region
 }
 
diff --git a/examples/disable_client_cert/main.tf b/examples/disable_client_cert/main.tf
index 329e1cf20..89f0a4712 100644
--- a/examples/disable_client_cert/main.tf
+++ b/examples/disable_client_cert/main.tf
@@ -19,7 +19,7 @@ locals {
 }
 
 provider "google" {
-  version = "~> 3.14.0"
+  version = "~> 3.16.0"
   region  = var.region
 }
 
diff --git a/examples/node_pool/main.tf b/examples/node_pool/main.tf
index 81681dd50..85da019a6 100644
--- a/examples/node_pool/main.tf
+++ b/examples/node_pool/main.tf
@@ -19,7 +19,7 @@ locals {
 }
 
 provider "google-beta" {
-  version = "~> 3.14.0"
+  version = "~> 3.16.0"
   region  = var.region
 }
 
diff --git a/examples/node_pool_update_variant/main.tf b/examples/node_pool_update_variant/main.tf
index 6225e631b..700ea9d5f 100644
--- a/examples/node_pool_update_variant/main.tf
+++ b/examples/node_pool_update_variant/main.tf
@@ -19,7 +19,7 @@ locals {
 }
 
 provider "google" {
-  version = "~> 3.14.0"
+  version = "~> 3.16.0"
   region  = var.region
 }
 
diff --git a/examples/node_pool_update_variant_beta/main.tf b/examples/node_pool_update_variant_beta/main.tf
index 0bd1b664d..8080c2b2a 100644
--- a/examples/node_pool_update_variant_beta/main.tf
+++ b/examples/node_pool_update_variant_beta/main.tf
@@ -19,7 +19,7 @@ locals {
 }
 
 provider "google-beta" {
-  version     = "~> 3.14.0"
+  version     = "~> 3.16.0"
   credentials = file(var.credentials_path)
   region      = var.region
 }
diff --git a/examples/regional_private_node_pool_oauth_scopes/provider.tf b/examples/regional_private_node_pool_oauth_scopes/provider.tf
index 11fd24e0e..76ebcc1a5 100644
--- a/examples/regional_private_node_pool_oauth_scopes/provider.tf
+++ b/examples/regional_private_node_pool_oauth_scopes/provider.tf
@@ -15,9 +15,9 @@
  */
 
 provider "google" {
-  version = "3.14.0"
+  version = "3.16.0"
 }
 
 provider "google-beta" {
-  version = "3.14.0"
+  version = "3.16.0"
 }
diff --git a/examples/safer_cluster/main.tf b/examples/safer_cluster/main.tf
index bf1328cd9..07e62ab12 100644
--- a/examples/safer_cluster/main.tf
+++ b/examples/safer_cluster/main.tf
@@ -30,11 +30,11 @@ locals {
 }
 
 provider "google" {
-  version = "~> 3.14.0"
+  version = "~> 3.16.0"
 }
 
 provider "google-beta" {
-  version = "~> 3.14.0"
+  version = "~> 3.16.0"
 }
 
 module "gke" {
diff --git a/examples/shared_vpc/main.tf b/examples/shared_vpc/main.tf
index 8c554fddb..0784c6f24 100644
--- a/examples/shared_vpc/main.tf
+++ b/examples/shared_vpc/main.tf
@@ -19,7 +19,7 @@ locals {
 }
 
 provider "google" {
-  version = "~> 3.14.0"
+  version = "~> 3.16.0"
   region  = var.region
 }
 
diff --git a/examples/simple_regional/main.tf b/examples/simple_regional/main.tf
index a03b1352f..70f03995a 100644
--- a/examples/simple_regional/main.tf
+++ b/examples/simple_regional/main.tf
@@ -19,7 +19,7 @@ locals {
 }
 
 provider "google" {
-  version = "~> 3.14.0"
+  version = "~> 3.16.0"
   region  = var.region
 }
 
diff --git a/examples/simple_regional_beta/main.tf b/examples/simple_regional_beta/main.tf
index 301c03adb..f994f77f0 100644
--- a/examples/simple_regional_beta/main.tf
+++ b/examples/simple_regional_beta/main.tf
@@ -19,7 +19,7 @@ locals {
 }
 
 provider "google-beta" {
-  version = "~> 3.14.0"
+  version = "~> 3.16.0"
   region  = var.region
 }
 
diff --git a/examples/simple_regional_private/main.tf b/examples/simple_regional_private/main.tf
index 271f12d9f..08d8f0959 100644
--- a/examples/simple_regional_private/main.tf
+++ b/examples/simple_regional_private/main.tf
@@ -19,7 +19,7 @@ locals {
 }
 
 provider "google" {
-  version = "~> 3.14.0"
+  version = "~> 3.16.0"
   region  = var.region
 }
 
diff --git a/examples/simple_regional_private_beta/main.tf b/examples/simple_regional_private_beta/main.tf
index 92f2a1a8d..ab0ac826f 100644
--- a/examples/simple_regional_private_beta/main.tf
+++ b/examples/simple_regional_private_beta/main.tf
@@ -19,12 +19,12 @@ locals {
 }
 
 provider "google" {
-  version = "~> 3.14.0"
+  version = "~> 3.16.0"
   region  = var.region
 }
 
 provider "google-beta" {
-  version = "~> 3.14.0"
+  version = "~> 3.16.0"
   region  = var.region
 }
 
diff --git a/examples/simple_regional_with_kubeconfig/main.tf b/examples/simple_regional_with_kubeconfig/main.tf
index 6aee4445f..c1a0a024d 100644
--- a/examples/simple_regional_with_kubeconfig/main.tf
+++ b/examples/simple_regional_with_kubeconfig/main.tf
@@ -19,7 +19,7 @@ locals {
 }
 
 provider "google" {
-  version = "~> 3.3.0"
+  version = "~> 3.16.0"
   region  = var.region
 }
 
diff --git a/examples/simple_regional_with_networking/main.tf b/examples/simple_regional_with_networking/main.tf
index a49079640..1eefe851a 100644
--- a/examples/simple_regional_with_networking/main.tf
+++ b/examples/simple_regional_with_networking/main.tf
@@ -15,7 +15,7 @@
  */
 
 provider "google" {
-  version = "~> 3.14.0"
+  version = "~> 3.16.0"
 }
 
 module "gcp-network" {
diff --git a/examples/simple_zonal_private/main.tf b/examples/simple_zonal_private/main.tf
index bb0581a83..492f39697 100644
--- a/examples/simple_zonal_private/main.tf
+++ b/examples/simple_zonal_private/main.tf
@@ -19,7 +19,7 @@ locals {
 }
 
 provider "google" {
-  version = "~> 3.14.0"
+  version = "~> 3.16.0"
   region  = var.region
 }
 
diff --git a/examples/simple_zonal_with_acm/main.tf b/examples/simple_zonal_with_acm/main.tf
index 40bb16645..94d48cc5e 100644
--- a/examples/simple_zonal_with_acm/main.tf
+++ b/examples/simple_zonal_with_acm/main.tf
@@ -19,7 +19,7 @@ locals {
 }
 
 provider "google" {
-  version = "~> 3.14.0"
+  version = "~> 3.16.0"
   region  = var.region
 }
 
diff --git a/examples/stub_domains/main.tf b/examples/stub_domains/main.tf
index ad7e17ebe..be779155a 100644
--- a/examples/stub_domains/main.tf
+++ b/examples/stub_domains/main.tf
@@ -19,7 +19,7 @@ locals {
 }
 
 provider "google" {
-  version = "~> 3.14.0"
+  version = "~> 3.16.0"
   region  = var.region
 }
 
diff --git a/examples/stub_domains_private/main.tf b/examples/stub_domains_private/main.tf
index a0022c478..2e8b6ca5c 100644
--- a/examples/stub_domains_private/main.tf
+++ b/examples/stub_domains_private/main.tf
@@ -15,7 +15,7 @@
  */
 
 provider "google" {
-  version = "~> 3.14.0"
+  version = "~> 3.16.0"
   region  = var.region
 }
 
diff --git a/examples/stub_domains_upstream_nameservers/main.tf b/examples/stub_domains_upstream_nameservers/main.tf
index a4d626c50..afdad4fa4 100644
--- a/examples/stub_domains_upstream_nameservers/main.tf
+++ b/examples/stub_domains_upstream_nameservers/main.tf
@@ -19,7 +19,7 @@ locals {
 }
 
 provider "google" {
-  version = "~> 3.14.0"
+  version = "~> 3.16.0"
   region  = var.region
 }
 
diff --git a/examples/upstream_nameservers/main.tf b/examples/upstream_nameservers/main.tf
index fb7d0581b..58207ce37 100644
--- a/examples/upstream_nameservers/main.tf
+++ b/examples/upstream_nameservers/main.tf
@@ -19,7 +19,7 @@ locals {
 }
 
 provider "google" {
-  version = "~> 3.14.0"
+  version = "~> 3.16.0"
   region  = var.region
 }
 
diff --git a/examples/workload_identity/main.tf b/examples/workload_identity/main.tf
index 6d810f6e3..9508195b9 100644
--- a/examples/workload_identity/main.tf
+++ b/examples/workload_identity/main.tf
@@ -19,7 +19,7 @@ locals {
 }
 
 provider "google" {
-  version = "~> 3.14.0"
+  version = "~> 3.16.0"
   region  = var.region
 }
 
diff --git a/examples/workload_metadata_config/main.tf b/examples/workload_metadata_config/main.tf
index 655c37619..9b40e16e3 100644
--- a/examples/workload_metadata_config/main.tf
+++ b/examples/workload_metadata_config/main.tf
@@ -19,7 +19,7 @@ locals {
 }
 
 provider "google-beta" {
-  version = "~> 3.14.0"
+  version = "~> 3.16.0"
   region  = var.region
 }
 
diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md
index 7940ba2a9..d8d301aa8 100644
--- a/modules/beta-private-cluster-update-variant/README.md
+++ b/modules/beta-private-cluster-update-variant/README.md
@@ -175,8 +175,10 @@ Then perform the following commands on the root folder:
 | enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no |
 | enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no |
 | enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | bool | `"false"` | no |
+| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | bool | `"false"` | no |
 | enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no |
 | enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no |
+| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | bool | `"true"` | no |
 | enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | bool | `"true"` | no |
 | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no |
 | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers | list(string) | `<list>` | no |
@@ -221,7 +223,7 @@ Then perform the following commands on the root folder:
 | registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
 | release\_channel | (Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"null"` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
-| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
+| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | string | `""` | no |
 | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | bool | `"false"` | no |
 | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
 | skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no |
diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf
index 53b22c716..21404796d 100644
--- a/modules/beta-private-cluster-update-variant/cluster.tf
+++ b/modules/beta-private-cluster-update-variant/cluster.tf
@@ -86,16 +86,6 @@ resource "google_container_cluster" "primary" {
       enabled = pod_security_policy_config.value.enabled
     }
   }
-
-  dynamic "resource_usage_export_config" {
-    for_each = var.resource_usage_export_dataset_id != "" ? [var.resource_usage_export_dataset_id] : []
-    content {
-      enable_network_egress_metering = true
-      bigquery_destination {
-        dataset_id = resource_usage_export_config.value
-      }
-    }
-  }
   dynamic "master_authorized_networks_config" {
     for_each = local.master_authorized_networks_config
     content {
@@ -199,6 +189,22 @@ resource "google_container_cluster" "primary" {
     }
   }
 
+  dynamic "resource_usage_export_config" {
+    for_each = var.resource_usage_export_dataset_id != "" ? [{
+      enable_network_egress_metering       = var.enable_network_egress_export
+      enable_resource_consumption_metering = var.enable_resource_consumption_export
+      dataset_id                           = var.resource_usage_export_dataset_id
+    }] : []
+
+    content {
+      enable_network_egress_metering       = resource_usage_export_config.value.enable_network_egress_metering
+      enable_resource_consumption_metering = resource_usage_export_config.value.enable_resource_consumption_metering
+      bigquery_destination {
+        dataset_id = resource_usage_export_config.value.dataset_id
+      }
+    }
+  }
+
   dynamic "private_cluster_config" {
     for_each = var.enable_private_nodes ? [{
       enable_private_nodes    = var.enable_private_nodes,
diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf
index aff8f1a62..0f3d3a8c4 100644
--- a/modules/beta-private-cluster-update-variant/variables.tf
+++ b/modules/beta-private-cluster-update-variant/variables.tf
@@ -181,6 +181,24 @@ variable "node_pools_metadata" {
   }
 }
 
+variable "resource_usage_export_dataset_id" {
+  type        = string
+  description = "The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export."
+  default     = ""
+}
+
+variable "enable_network_egress_export" {
+  type        = bool
+  description = "Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic."
+  default     = false
+}
+
+variable "enable_resource_consumption_export" {
+  type        = bool
+  description = "Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export."
+  default     = true
+}
+
 variable "enable_kubernetes_alpha" {
   type        = bool
   description = "Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days."
@@ -421,12 +439,6 @@ variable "pod_security_policy_config" {
   }]
 }
 
-variable "resource_usage_export_dataset_id" {
-  type        = string
-  description = "The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic."
-  default     = ""
-}
-
 variable "node_metadata" {
   description = "Specifies how node metadata is exposed to the workload running on the node"
   default     = "SECURE"
diff --git a/modules/beta-private-cluster-update-variant/versions.tf b/modules/beta-private-cluster-update-variant/versions.tf
index 7ce903755..49108e267 100644
--- a/modules/beta-private-cluster-update-variant/versions.tf
+++ b/modules/beta-private-cluster-update-variant/versions.tf
@@ -18,6 +18,6 @@ terraform {
   required_version = "~> 0.12.6"
 
   required_providers {
-    google-beta = ">= 3.1, <4.0.0"
+    google-beta = ">= 3.16, <4.0.0"
   }
 }
diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md
index a73e37ec2..bf11a3eee 100644
--- a/modules/beta-private-cluster/README.md
+++ b/modules/beta-private-cluster/README.md
@@ -153,8 +153,10 @@ Then perform the following commands on the root folder:
 | enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no |
 | enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no |
 | enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | bool | `"false"` | no |
+| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | bool | `"false"` | no |
 | enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no |
 | enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no |
+| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | bool | `"true"` | no |
 | enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | bool | `"true"` | no |
 | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no |
 | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers | list(string) | `<list>` | no |
@@ -199,7 +201,7 @@ Then perform the following commands on the root folder:
 | registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
 | release\_channel | (Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"null"` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
-| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
+| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | string | `""` | no |
 | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | bool | `"false"` | no |
 | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
 | skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no |
diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf
index 98c3a7362..60a686564 100644
--- a/modules/beta-private-cluster/cluster.tf
+++ b/modules/beta-private-cluster/cluster.tf
@@ -86,16 +86,6 @@ resource "google_container_cluster" "primary" {
       enabled = pod_security_policy_config.value.enabled
     }
   }
-
-  dynamic "resource_usage_export_config" {
-    for_each = var.resource_usage_export_dataset_id != "" ? [var.resource_usage_export_dataset_id] : []
-    content {
-      enable_network_egress_metering = true
-      bigquery_destination {
-        dataset_id = resource_usage_export_config.value
-      }
-    }
-  }
   dynamic "master_authorized_networks_config" {
     for_each = local.master_authorized_networks_config
     content {
@@ -199,6 +189,22 @@ resource "google_container_cluster" "primary" {
     }
   }
 
+  dynamic "resource_usage_export_config" {
+    for_each = var.resource_usage_export_dataset_id != "" ? [{
+      enable_network_egress_metering       = var.enable_network_egress_export
+      enable_resource_consumption_metering = var.enable_resource_consumption_export
+      dataset_id                           = var.resource_usage_export_dataset_id
+    }] : []
+
+    content {
+      enable_network_egress_metering       = resource_usage_export_config.value.enable_network_egress_metering
+      enable_resource_consumption_metering = resource_usage_export_config.value.enable_resource_consumption_metering
+      bigquery_destination {
+        dataset_id = resource_usage_export_config.value.dataset_id
+      }
+    }
+  }
+
   dynamic "private_cluster_config" {
     for_each = var.enable_private_nodes ? [{
       enable_private_nodes    = var.enable_private_nodes,
diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf
index aff8f1a62..0f3d3a8c4 100644
--- a/modules/beta-private-cluster/variables.tf
+++ b/modules/beta-private-cluster/variables.tf
@@ -181,6 +181,24 @@ variable "node_pools_metadata" {
   }
 }
 
+variable "resource_usage_export_dataset_id" {
+  type        = string
+  description = "The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export."
+  default     = ""
+}
+
+variable "enable_network_egress_export" {
+  type        = bool
+  description = "Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic."
+  default     = false
+}
+
+variable "enable_resource_consumption_export" {
+  type        = bool
+  description = "Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export."
+  default     = true
+}
+
 variable "enable_kubernetes_alpha" {
   type        = bool
   description = "Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days."
@@ -421,12 +439,6 @@ variable "pod_security_policy_config" {
   }]
 }
 
-variable "resource_usage_export_dataset_id" {
-  type        = string
-  description = "The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic."
-  default     = ""
-}
-
 variable "node_metadata" {
   description = "Specifies how node metadata is exposed to the workload running on the node"
   default     = "SECURE"
diff --git a/modules/beta-private-cluster/versions.tf b/modules/beta-private-cluster/versions.tf
index 7ce903755..49108e267 100644
--- a/modules/beta-private-cluster/versions.tf
+++ b/modules/beta-private-cluster/versions.tf
@@ -18,6 +18,6 @@ terraform {
   required_version = "~> 0.12.6"
 
   required_providers {
-    google-beta = ">= 3.1, <4.0.0"
+    google-beta = ">= 3.16, <4.0.0"
   }
 }
diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md
index 3a4ad3795..8b99702b4 100644
--- a/modules/beta-public-cluster/README.md
+++ b/modules/beta-public-cluster/README.md
@@ -134,6 +134,8 @@ Then perform the following commands on the root folder:
 | enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no |
 | enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no |
 | enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | bool | `"false"` | no |
+| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | bool | `"false"` | no |
+| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | bool | `"true"` | no |
 | enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | bool | `"true"` | no |
 | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no |
 | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers | list(string) | `<list>` | no |
@@ -177,7 +179,7 @@ Then perform the following commands on the root folder:
 | registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
 | release\_channel | (Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"null"` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
-| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
+| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | string | `""` | no |
 | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | bool | `"false"` | no |
 | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
 | skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no |
diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf
index 3d8678a05..20dd350de 100644
--- a/modules/beta-public-cluster/cluster.tf
+++ b/modules/beta-public-cluster/cluster.tf
@@ -86,16 +86,6 @@ resource "google_container_cluster" "primary" {
       enabled = pod_security_policy_config.value.enabled
     }
   }
-
-  dynamic "resource_usage_export_config" {
-    for_each = var.resource_usage_export_dataset_id != "" ? [var.resource_usage_export_dataset_id] : []
-    content {
-      enable_network_egress_metering = true
-      bigquery_destination {
-        dataset_id = resource_usage_export_config.value
-      }
-    }
-  }
   dynamic "master_authorized_networks_config" {
     for_each = local.master_authorized_networks_config
     content {
@@ -199,6 +189,22 @@ resource "google_container_cluster" "primary" {
     }
   }
 
+  dynamic "resource_usage_export_config" {
+    for_each = var.resource_usage_export_dataset_id != "" ? [{
+      enable_network_egress_metering       = var.enable_network_egress_export
+      enable_resource_consumption_metering = var.enable_resource_consumption_export
+      dataset_id                           = var.resource_usage_export_dataset_id
+    }] : []
+
+    content {
+      enable_network_egress_metering       = resource_usage_export_config.value.enable_network_egress_metering
+      enable_resource_consumption_metering = resource_usage_export_config.value.enable_resource_consumption_metering
+      bigquery_destination {
+        dataset_id = resource_usage_export_config.value.dataset_id
+      }
+    }
+  }
+
 
   remove_default_node_pool = var.remove_default_node_pool
 
diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf
index 2bb84a705..2d802775e 100644
--- a/modules/beta-public-cluster/variables.tf
+++ b/modules/beta-public-cluster/variables.tf
@@ -181,6 +181,24 @@ variable "node_pools_metadata" {
   }
 }
 
+variable "resource_usage_export_dataset_id" {
+  type        = string
+  description = "The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export."
+  default     = ""
+}
+
+variable "enable_network_egress_export" {
+  type        = bool
+  description = "Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic."
+  default     = false
+}
+
+variable "enable_resource_consumption_export" {
+  type        = bool
+  description = "Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export."
+  default     = true
+}
+
 variable "enable_kubernetes_alpha" {
   type        = bool
   description = "Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days."
@@ -397,12 +415,6 @@ variable "pod_security_policy_config" {
   }]
 }
 
-variable "resource_usage_export_dataset_id" {
-  type        = string
-  description = "The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic."
-  default     = ""
-}
-
 variable "node_metadata" {
   description = "Specifies how node metadata is exposed to the workload running on the node"
   default     = "SECURE"
diff --git a/modules/beta-public-cluster/versions.tf b/modules/beta-public-cluster/versions.tf
index 7ce903755..49108e267 100644
--- a/modules/beta-public-cluster/versions.tf
+++ b/modules/beta-public-cluster/versions.tf
@@ -18,6 +18,6 @@ terraform {
   required_version = "~> 0.12.6"
 
   required_providers {
-    google-beta = ">= 3.1, <4.0.0"
+    google-beta = ">= 3.16, <4.0.0"
   }
 }
diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md
index 9fb0dfe5b..b96bb24d2 100644
--- a/modules/private-cluster-update-variant/README.md
+++ b/modules/private-cluster-update-variant/README.md
@@ -151,8 +151,10 @@ Then perform the following commands on the root folder:
 | deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no |
 | description | The description of the cluster | string | `""` | no |
 | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no |
+| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | bool | `"false"` | no |
 | enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no |
 | enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no |
+| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | bool | `"true"` | no |
 | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers | list(string) | `<list>` | no |
 | firewall\_priority | Priority rule for firewall rules | number | `"1000"` | no |
 | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
@@ -186,6 +188,7 @@ Then perform the following commands on the root folder:
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
 | registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
+| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | string | `""` | no |
 | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
 | skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no |
 | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `<map>` | no |
diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf
index 7128c2de4..b084990f3 100644
--- a/modules/private-cluster-update-variant/cluster.tf
+++ b/modules/private-cluster-update-variant/cluster.tf
@@ -118,6 +118,22 @@ resource "google_container_cluster" "primary" {
     }
   }
 
+  dynamic "resource_usage_export_config" {
+    for_each = var.resource_usage_export_dataset_id != "" ? [{
+      enable_network_egress_metering       = var.enable_network_egress_export
+      enable_resource_consumption_metering = var.enable_resource_consumption_export
+      dataset_id                           = var.resource_usage_export_dataset_id
+    }] : []
+
+    content {
+      enable_network_egress_metering       = resource_usage_export_config.value.enable_network_egress_metering
+      enable_resource_consumption_metering = resource_usage_export_config.value.enable_resource_consumption_metering
+      bigquery_destination {
+        dataset_id = resource_usage_export_config.value.dataset_id
+      }
+    }
+  }
+
   dynamic "private_cluster_config" {
     for_each = var.enable_private_nodes ? [{
       enable_private_nodes    = var.enable_private_nodes,
diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf
index 7ad0f86ba..0d76d967b 100644
--- a/modules/private-cluster-update-variant/variables.tf
+++ b/modules/private-cluster-update-variant/variables.tf
@@ -169,6 +169,24 @@ variable "node_pools_metadata" {
     default-node-pool = {}
   }
 }
+
+variable "resource_usage_export_dataset_id" {
+  type        = string
+  description = "The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export."
+  default     = ""
+}
+
+variable "enable_network_egress_export" {
+  type        = bool
+  description = "Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic."
+  default     = false
+}
+
+variable "enable_resource_consumption_export" {
+  type        = bool
+  description = "Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export."
+  default     = true
+}
 variable "node_pools_tags" {
   type        = map(list(string))
   description = "Map of lists containing node network tags by node-pool name"
diff --git a/modules/private-cluster-update-variant/versions.tf b/modules/private-cluster-update-variant/versions.tf
index 1bdc2f911..7b6c516d4 100644
--- a/modules/private-cluster-update-variant/versions.tf
+++ b/modules/private-cluster-update-variant/versions.tf
@@ -18,6 +18,6 @@ terraform {
   required_version = "~> 0.12.6"
 
   required_providers {
-    google = ">= 2.18, <4.0.0"
+    google = ">= 3.16, <4.0.0"
   }
 }
diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md
index 0c23d6109..55bd48bc0 100644
--- a/modules/private-cluster/README.md
+++ b/modules/private-cluster/README.md
@@ -129,8 +129,10 @@ Then perform the following commands on the root folder:
 | deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no |
 | description | The description of the cluster | string | `""` | no |
 | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no |
+| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | bool | `"false"` | no |
 | enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no |
 | enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no |
+| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | bool | `"true"` | no |
 | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers | list(string) | `<list>` | no |
 | firewall\_priority | Priority rule for firewall rules | number | `"1000"` | no |
 | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
@@ -164,6 +166,7 @@ Then perform the following commands on the root folder:
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
 | registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
+| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | string | `""` | no |
 | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
 | skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no |
 | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `<map>` | no |
diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf
index 75dbeb36f..ef943467e 100644
--- a/modules/private-cluster/cluster.tf
+++ b/modules/private-cluster/cluster.tf
@@ -118,6 +118,22 @@ resource "google_container_cluster" "primary" {
     }
   }
 
+  dynamic "resource_usage_export_config" {
+    for_each = var.resource_usage_export_dataset_id != "" ? [{
+      enable_network_egress_metering       = var.enable_network_egress_export
+      enable_resource_consumption_metering = var.enable_resource_consumption_export
+      dataset_id                           = var.resource_usage_export_dataset_id
+    }] : []
+
+    content {
+      enable_network_egress_metering       = resource_usage_export_config.value.enable_network_egress_metering
+      enable_resource_consumption_metering = resource_usage_export_config.value.enable_resource_consumption_metering
+      bigquery_destination {
+        dataset_id = resource_usage_export_config.value.dataset_id
+      }
+    }
+  }
+
   dynamic "private_cluster_config" {
     for_each = var.enable_private_nodes ? [{
       enable_private_nodes    = var.enable_private_nodes,
diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf
index 7ad0f86ba..0d76d967b 100644
--- a/modules/private-cluster/variables.tf
+++ b/modules/private-cluster/variables.tf
@@ -169,6 +169,24 @@ variable "node_pools_metadata" {
     default-node-pool = {}
   }
 }
+
+variable "resource_usage_export_dataset_id" {
+  type        = string
+  description = "The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export."
+  default     = ""
+}
+
+variable "enable_network_egress_export" {
+  type        = bool
+  description = "Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic."
+  default     = false
+}
+
+variable "enable_resource_consumption_export" {
+  type        = bool
+  description = "Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export."
+  default     = true
+}
 variable "node_pools_tags" {
   type        = map(list(string))
   description = "Map of lists containing node network tags by node-pool name"
diff --git a/modules/private-cluster/versions.tf b/modules/private-cluster/versions.tf
index 1bdc2f911..7b6c516d4 100644
--- a/modules/private-cluster/versions.tf
+++ b/modules/private-cluster/versions.tf
@@ -18,6 +18,6 @@ terraform {
   required_version = "~> 0.12.6"
 
   required_providers {
-    google = ">= 2.18, <4.0.0"
+    google = ">= 3.16, <4.0.0"
   }
 }
diff --git a/modules/safer-cluster-update-variant/README.md b/modules/safer-cluster-update-variant/README.md
index 7853a18eb..1ef75b926 100644
--- a/modules/safer-cluster-update-variant/README.md
+++ b/modules/safer-cluster-update-variant/README.md
@@ -209,7 +209,9 @@ For simplicity, we suggest using `roles/container.admin` and
 | description | The description of the cluster | string | `""` | no |
 | dns\_cache | (Beta) The status of the NodeLocal DNSCache addon. | bool | `"false"` | no |
 | enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no |
+| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | bool | `"false"` | no |
 | enable\_private\_endpoint | When true, the cluster's private endpoint is used as the cluster endpoint and access through the public endpoint is disabled. When false, either endpoint can be used. This field only applies to private clusters, when enable_private_nodes is true | bool | `"true"` | no |
+| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | bool | `"true"` | no |
 | enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster. | bool | `"true"` | no |
 | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no |
 | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
diff --git a/modules/safer-cluster-update-variant/variables.tf b/modules/safer-cluster-update-variant/variables.tf
index 848b76830..02cbdb084 100644
--- a/modules/safer-cluster-update-variant/variables.tf
+++ b/modules/safer-cluster-update-variant/variables.tf
@@ -268,6 +268,18 @@ variable "resource_usage_export_dataset_id" {
   default     = ""
 }
 
+variable "enable_network_egress_export" {
+  type        = bool
+  description = "Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic."
+  default     = false
+}
+
+variable "enable_resource_consumption_export" {
+  type        = bool
+  description = "Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export."
+  default     = true
+}
+
 variable "sandbox_enabled" {
   type        = bool
   description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it)."
diff --git a/modules/safer-cluster/README.md b/modules/safer-cluster/README.md
index 7853a18eb..1ef75b926 100644
--- a/modules/safer-cluster/README.md
+++ b/modules/safer-cluster/README.md
@@ -209,7 +209,9 @@ For simplicity, we suggest using `roles/container.admin` and
 | description | The description of the cluster | string | `""` | no |
 | dns\_cache | (Beta) The status of the NodeLocal DNSCache addon. | bool | `"false"` | no |
 | enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no |
+| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | bool | `"false"` | no |
 | enable\_private\_endpoint | When true, the cluster's private endpoint is used as the cluster endpoint and access through the public endpoint is disabled. When false, either endpoint can be used. This field only applies to private clusters, when enable_private_nodes is true | bool | `"true"` | no |
+| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | bool | `"true"` | no |
 | enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster. | bool | `"true"` | no |
 | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no |
 | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
diff --git a/modules/safer-cluster/variables.tf b/modules/safer-cluster/variables.tf
index 848b76830..02cbdb084 100644
--- a/modules/safer-cluster/variables.tf
+++ b/modules/safer-cluster/variables.tf
@@ -268,6 +268,18 @@ variable "resource_usage_export_dataset_id" {
   default     = ""
 }
 
+variable "enable_network_egress_export" {
+  type        = bool
+  description = "Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic."
+  default     = false
+}
+
+variable "enable_resource_consumption_export" {
+  type        = bool
+  description = "Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export."
+  default     = true
+}
+
 variable "sandbox_enabled" {
   type        = bool
   description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it)."
diff --git a/test/fixtures/deploy_service/network.tf b/test/fixtures/deploy_service/network.tf
index f5bad4f28..513747de5 100644
--- a/test/fixtures/deploy_service/network.tf
+++ b/test/fixtures/deploy_service/network.tf
@@ -21,7 +21,7 @@ resource "random_string" "suffix" {
 }
 
 provider "google" {
-  version = "~> 3.14.0"
+  version = "~> 3.16.0"
   project = var.project_ids[0]
 }
 
diff --git a/test/fixtures/disable_client_cert/network.tf b/test/fixtures/disable_client_cert/network.tf
index f5bad4f28..513747de5 100644
--- a/test/fixtures/disable_client_cert/network.tf
+++ b/test/fixtures/disable_client_cert/network.tf
@@ -21,7 +21,7 @@ resource "random_string" "suffix" {
 }
 
 provider "google" {
-  version = "~> 3.14.0"
+  version = "~> 3.16.0"
   project = var.project_ids[0]
 }
 
diff --git a/test/fixtures/shared_vpc/network.tf b/test/fixtures/shared_vpc/network.tf
index f5bad4f28..513747de5 100644
--- a/test/fixtures/shared_vpc/network.tf
+++ b/test/fixtures/shared_vpc/network.tf
@@ -21,7 +21,7 @@ resource "random_string" "suffix" {
 }
 
 provider "google" {
-  version = "~> 3.14.0"
+  version = "~> 3.16.0"
   project = var.project_ids[0]
 }
 
diff --git a/test/fixtures/simple_regional/network.tf b/test/fixtures/simple_regional/network.tf
index f5bad4f28..513747de5 100644
--- a/test/fixtures/simple_regional/network.tf
+++ b/test/fixtures/simple_regional/network.tf
@@ -21,7 +21,7 @@ resource "random_string" "suffix" {
 }
 
 provider "google" {
-  version = "~> 3.14.0"
+  version = "~> 3.16.0"
   project = var.project_ids[0]
 }
 
diff --git a/test/fixtures/simple_regional_with_kubeconfig/network.tf b/test/fixtures/simple_regional_with_kubeconfig/network.tf
index fad61a918..75f23bea6 100644
--- a/test/fixtures/simple_regional_with_kubeconfig/network.tf
+++ b/test/fixtures/simple_regional_with_kubeconfig/network.tf
@@ -21,7 +21,7 @@ resource "random_string" "suffix" {
 }
 
 provider "google" {
-  version = "~> 3.3.0"
+  version = "~> 3.16.0"
   project = var.project_ids[0]
 }
 
diff --git a/test/fixtures/simple_zonal/network.tf b/test/fixtures/simple_zonal/network.tf
index f0d617ec7..c4c25d587 100644
--- a/test/fixtures/simple_zonal/network.tf
+++ b/test/fixtures/simple_zonal/network.tf
@@ -21,7 +21,7 @@ resource "random_string" "suffix" {
 }
 
 provider "google" {
-  version = "~> 3.14.0"
+  version = "~> 3.16.0"
   project = var.project_ids[1]
 }
 
diff --git a/test/fixtures/stub_domains/network.tf b/test/fixtures/stub_domains/network.tf
index 3f6313acb..5a8d36e7c 100644
--- a/test/fixtures/stub_domains/network.tf
+++ b/test/fixtures/stub_domains/network.tf
@@ -21,7 +21,7 @@ resource "random_string" "suffix" {
 }
 
 provider "google" {
-  version = "~> 3.14.0"
+  version = "~> 3.16.0"
   project = var.project_ids[1]
 }
 
diff --git a/test/fixtures/stub_domains_upstream_nameservers/network.tf b/test/fixtures/stub_domains_upstream_nameservers/network.tf
index a1affbb51..a1f546be2 100644
--- a/test/fixtures/stub_domains_upstream_nameservers/network.tf
+++ b/test/fixtures/stub_domains_upstream_nameservers/network.tf
@@ -21,7 +21,7 @@ resource "random_string" "suffix" {
 }
 
 provider "google" {
-  version = "~> 3.14.0"
+  version = "~> 3.16.0"
   project = var.project_ids[1]
 }
 
diff --git a/test/fixtures/upstream_nameservers/network.tf b/test/fixtures/upstream_nameservers/network.tf
index a1affbb51..a1f546be2 100644
--- a/test/fixtures/upstream_nameservers/network.tf
+++ b/test/fixtures/upstream_nameservers/network.tf
@@ -21,7 +21,7 @@ resource "random_string" "suffix" {
 }
 
 provider "google" {
-  version = "~> 3.14.0"
+  version = "~> 3.16.0"
   project = var.project_ids[1]
 }
 
diff --git a/variables.tf b/variables.tf
index 5f8da02b5..41a569ce8 100644
--- a/variables.tf
+++ b/variables.tf
@@ -169,6 +169,24 @@ variable "node_pools_metadata" {
     default-node-pool = {}
   }
 }
+
+variable "resource_usage_export_dataset_id" {
+  type        = string
+  description = "The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export."
+  default     = ""
+}
+
+variable "enable_network_egress_export" {
+  type        = bool
+  description = "Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic."
+  default     = false
+}
+
+variable "enable_resource_consumption_export" {
+  type        = bool
+  description = "Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export."
+  default     = true
+}
 variable "node_pools_tags" {
   type        = map(list(string))
   description = "Map of lists containing node network tags by node-pool name"
diff --git a/versions.tf b/versions.tf
index 1bdc2f911..7b6c516d4 100644
--- a/versions.tf
+++ b/versions.tf
@@ -18,6 +18,6 @@ terraform {
   required_version = "~> 0.12.6"
 
   required_providers {
-    google = ">= 2.18, <4.0.0"
+    google = ">= 3.16, <4.0.0"
   }
 }