diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl index 426ba4c30..663d9a8e9 100644 --- a/autogen/main/cluster.tf.tmpl +++ b/autogen/main/cluster.tf.tmpl @@ -258,10 +258,10 @@ resource "google_container_cluster" "primary" { } {% if autopilot_cluster %} dynamic "node_pool_auto_config" { - for_each = length(var.network_tags) > 0 ? [1] : [] + for_each = length(var.network_tags) > 0 || var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? [1] : [] content { network_tags { - tags = var.network_tags + tags = var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? concat(var.network_tags, [local.cluster_network_tag]) : var.network_tags } } } diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 8dc74732a..13306c87b 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -454,6 +454,27 @@ steps: - verify simple-autopilot-private-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestSimpleAutopilotPrivate --stage teardown --verbose'] +- id: init autopilot-private-firewalls + waitFor: + - create-all + - destroy simple-autopilot-private-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestAutopilotPrivateFirewalls --stage init --verbose'] +- id: apply autopilot-private-firewalls + waitFor: + - init autopilot-private-firewalls + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestAutopilotPrivateFirewalls --stage apply --verbose'] +- id: verify autopilot-private-firewalls + waitFor: + - apply autopilot-private-firewalls + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestAutopilotPrivateFirewalls --stage verify --verbose'] +- id: destroy autopilot-private-firewalls + waitFor: + - verify autopilot-private-firewalls + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestAutopilotPrivateFirewalls --stage destroy --verbose'] - id: init simple-autopilot-public-local waitFor: - create-all diff --git a/examples/autopilot_private_firewalls/README.md b/examples/autopilot_private_firewalls/README.md new file mode 100644 index 000000000..1ee0a0ba3 --- /dev/null +++ b/examples/autopilot_private_firewalls/README.md @@ -0,0 +1,42 @@ +# Private Regional Autopilot Cluster With Firewall Rules + +This example creates a regional autopilot cluster with beta features + +It will: +- Create a private autopilot cluster +- Variables ```add_cluster_firewall_rules```, ```add_master_webhook_firewall_rules``` and ```add_shadow_firewal_rules``` are toggled on. In a environment with stringent firewall rules, these cluster firewall rules may be required to allow intra cluster communication +- Adds an example network tag. This example network tag aligns with a firewall rule target tag from the [terraform-example-foundation](https://github.com/terraform-google-modules/terraform-example-foundation) configuration to allow google api access + +>note: this example does **not** create a network with stringent firewall rules. This example shows what cluster configuration may be required for a networking configuration comparable to the [terraform-example-foundation](https://github.com/terraform-google-modules/terraform-example-foundation)'s base shared VPCs, or any VPC with firewall rules to deny 0.0.0.0/0 egress and an allow rule for private google api egress access which targets tags "allow-google-apis". By toggling on the firewall rules variables and adding network_tags to allow private google api access, a private cluster can come up healthy with no internet egress. This configuration shows how to ensure those firewalls that explicitly allow intra cluster ingress and egress are created and appropriate network tags are attached to the cluster. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| project\_id | The project ID to host the cluster in | `any` | n/a | yes | +| region | The region the cluster in | `string` | `"us-central1"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | The cluster ca certificate (base64 encoded) | +| cluster\_name | Cluster name | +| kubernetes\_endpoint | The cluster endpoint | +| location | n/a | +| master\_kubernetes\_version | Kubernetes version of the master | +| network\_name | The name of the VPC being created | +| project\_id | The project ID the cluster is in | +| region | The region in which the cluster resides | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| subnet\_names | The names of the subnet being created | +| zones | List of zones in which the cluster resides | + + + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/autopilot_private_firewalls/main.tf b/examples/autopilot_private_firewalls/main.tf new file mode 100644 index 000000000..979261c99 --- /dev/null +++ b/examples/autopilot_private_firewalls/main.tf @@ -0,0 +1,63 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cluster_type = "autopilot-private-firewalls" + network_name = "${local.cluster_type}-network" + subnet_name = "${local.cluster_type}-subnet" + master_auth_subnetwork = "${local.cluster_type}-master-subnet" + pods_range_name = "ip-range-pods-${local.cluster_type}" + svc_range_name = "ip-range-svc-${local.cluster_type}" + subnet_names = [for subnet_self_link in module.gcp-network.subnets_self_links : split("/", subnet_self_link)[length(split("/", subnet_self_link)) - 1]] +} + +data "google_client_config" "default" {} + +provider "kubernetes" { + host = "https://${module.gke.endpoint}" + token = data.google_client_config.default.access_token + cluster_ca_certificate = base64decode(module.gke.ca_certificate) +} + +module "gke" { + source = "../../modules/beta-autopilot-private-cluster/" + project_id = var.project_id + name = "${local.cluster_type}-cluster" + regional = true + region = var.region + network = module.gcp-network.network_name + subnetwork = local.subnet_names[index(module.gcp-network.subnets_names, local.subnet_name)] + ip_range_pods = local.pods_range_name + ip_range_services = local.svc_range_name + release_channel = "REGULAR" + enable_vertical_pod_autoscaling = true + enable_private_endpoint = true + enable_private_nodes = true + master_ipv4_cidr_block = "172.16.0.0/28" + add_cluster_firewall_rules = true + add_master_webhook_firewall_rules = true + add_shadow_firewall_rules = true + network_tags = ["allow-google-apis"] + deletion_protection = false + + master_authorized_networks = [ + { + cidr_block = "10.60.0.0/17" + display_name = "VPC" + }, + ] +} + diff --git a/examples/autopilot_private_firewalls/network.tf b/examples/autopilot_private_firewalls/network.tf new file mode 100644 index 000000000..acb96e028 --- /dev/null +++ b/examples/autopilot_private_firewalls/network.tf @@ -0,0 +1,49 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "gcp-network" { + source = "terraform-google-modules/network/google" + version = ">= 9.0.0" + + project_id = var.project_id + network_name = local.network_name + + subnets = [ + { + subnet_name = local.subnet_name + subnet_ip = "10.0.0.0/17" + subnet_region = var.region + }, + { + subnet_name = local.master_auth_subnetwork + subnet_ip = "10.60.0.0/17" + subnet_region = var.region + }, + ] + + secondary_ranges = { + (local.subnet_name) = [ + { + range_name = local.pods_range_name + ip_cidr_range = "192.168.0.0/18" + }, + { + range_name = local.svc_range_name + ip_cidr_range = "192.168.64.0/18" + }, + ] + } +} diff --git a/examples/autopilot_private_firewalls/outputs.tf b/examples/autopilot_private_firewalls/outputs.tf new file mode 100644 index 000000000..9691215da --- /dev/null +++ b/examples/autopilot_private_firewalls/outputs.tf @@ -0,0 +1,71 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "kubernetes_endpoint" { + description = "The cluster endpoint" + sensitive = true + value = module.gke.endpoint +} + +output "cluster_name" { + description = "Cluster name" + value = module.gke.name +} + +output "location" { + value = module.gke.location +} + +output "master_kubernetes_version" { + description = "Kubernetes version of the master" + value = module.gke.master_version +} + +output "ca_certificate" { + sensitive = true + description = "The cluster ca certificate (base64 encoded)" + value = module.gke.ca_certificate +} + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = module.gke.service_account +} + +output "network_name" { + description = "The name of the VPC being created" + value = module.gcp-network.network_name +} + +output "subnet_names" { + description = "The names of the subnet being created" + value = module.gcp-network.subnets_names +} + +output "region" { + description = "The region in which the cluster resides" + value = module.gke.region +} + +output "zones" { + description = "List of zones in which the cluster resides" + value = module.gke.zones +} + +output "project_id" { + description = "The project ID the cluster is in" + value = var.project_id +} diff --git a/examples/autopilot_private_firewalls/variables.tf b/examples/autopilot_private_firewalls/variables.tf new file mode 100644 index 000000000..25d63aeb4 --- /dev/null +++ b/examples/autopilot_private_firewalls/variables.tf @@ -0,0 +1,24 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID to host the cluster in" +} + +variable "region" { + description = "The region the cluster in" + default = "us-central1" +} diff --git a/examples/autopilot_private_firewalls/versions.tf b/examples/autopilot_private_firewalls/versions.tf new file mode 100644 index 000000000..7e6d0ea44 --- /dev/null +++ b/examples/autopilot_private_firewalls/versions.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + kubernetes = { + source = "hashicorp/kubernetes" + } + } + required_version = ">= 0.13" +} diff --git a/modules/beta-autopilot-private-cluster/cluster.tf b/modules/beta-autopilot-private-cluster/cluster.tf index 397c7755b..4a08c7e25 100644 --- a/modules/beta-autopilot-private-cluster/cluster.tf +++ b/modules/beta-autopilot-private-cluster/cluster.tf @@ -99,10 +99,10 @@ resource "google_container_cluster" "primary" { } } dynamic "node_pool_auto_config" { - for_each = length(var.network_tags) > 0 ? [1] : [] + for_each = length(var.network_tags) > 0 || var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? [1] : [] content { network_tags { - tags = var.network_tags + tags = var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? concat(var.network_tags, [local.cluster_network_tag]) : var.network_tags } } } diff --git a/modules/beta-autopilot-public-cluster/cluster.tf b/modules/beta-autopilot-public-cluster/cluster.tf index 01ea54baf..9d9573fd3 100644 --- a/modules/beta-autopilot-public-cluster/cluster.tf +++ b/modules/beta-autopilot-public-cluster/cluster.tf @@ -99,10 +99,10 @@ resource "google_container_cluster" "primary" { } } dynamic "node_pool_auto_config" { - for_each = length(var.network_tags) > 0 ? [1] : [] + for_each = length(var.network_tags) > 0 || var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? [1] : [] content { network_tags { - tags = var.network_tags + tags = var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? concat(var.network_tags, [local.cluster_network_tag]) : var.network_tags } } } diff --git a/test/fixtures/autopilot_private_firewalls/example.tf b/test/fixtures/autopilot_private_firewalls/example.tf new file mode 100644 index 000000000..7c7a7bbe0 --- /dev/null +++ b/test/fixtures/autopilot_private_firewalls/example.tf @@ -0,0 +1,22 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "example" { + source = "../../../examples/autopilot_private_firewalls" + + project_id = var.project_ids[0] + region = var.region +} diff --git a/test/fixtures/autopilot_private_firewalls/outputs.tf b/test/fixtures/autopilot_private_firewalls/outputs.tf new file mode 100644 index 000000000..f453fce01 --- /dev/null +++ b/test/fixtures/autopilot_private_firewalls/outputs.tf @@ -0,0 +1,52 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "project_id" { + value = module.example.project_id +} + +output "region" { + value = module.example.region +} + +output "cluster_name" { + description = "Cluster name" + value = module.example.cluster_name +} +output "location" { + value = module.example.location +} + +output "master_kubernetes_version" { + description = "The master Kubernetes version" + value = module.example.master_kubernetes_version +} + +output "kubernetes_endpoint" { + sensitive = true + value = module.example.kubernetes_endpoint +} + +output "ca_certificate" { + description = "The cluster CA certificate" + value = module.example.ca_certificate + sensitive = true +} + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = module.example.service_account +} diff --git a/test/fixtures/autopilot_private_firewalls/variables.tf b/test/fixtures/autopilot_private_firewalls/variables.tf new file mode 100644 index 000000000..b451d85cc --- /dev/null +++ b/test/fixtures/autopilot_private_firewalls/variables.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_ids" { + type = list(string) + description = "The GCP projects to use for integration tests" +} + +variable "region" { + description = "The GCP region to create and test resources in" + default = "us-central1" +} diff --git a/test/integration/autopilot_private_firewalls/autopilot_private_firewalls_test.go b/test/integration/autopilot_private_firewalls/autopilot_private_firewalls_test.go new file mode 100644 index 000000000..8ba773fbc --- /dev/null +++ b/test/integration/autopilot_private_firewalls/autopilot_private_firewalls_test.go @@ -0,0 +1,72 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package autopilot_private_firewalls + +import ( + "fmt" + "testing" + + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/golden" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" + "github.com/stretchr/testify/assert" +) + +func TestAutopilotPrivateFirewalls(t *testing.T) { + bpt := tft.NewTFBlueprintTest(t) + bpt.DefineVerify(func(assert *assert.Assertions) { + //Skipping Default Verify as the Verify Stage fails due to change in Client Cert Token + // bpt.DefaultVerify(assert) + projectId := bpt.GetStringOutput("project_id") + location := bpt.GetStringOutput("location") + clusterName := bpt.GetStringOutput("cluster_name") + serviceAccount := bpt.GetStringOutput("service_account") + clusterNetworkTag := "gke-" + clusterName + firewallRules := []string{"gke-%s-intra-cluster-egress", "gke-%s-webhooks", "gke-shadow-%s-all", "gke-shadow-%s-master", "gke-shadow-%s-vms", "gke-shadow-%s-inkubelet", "gke-shadow-%s-exkubelet"} + var fws []string + for _, fw := range firewallRules { + n := fmt.Sprintf(fw, clusterName) + fws = append(fws, n) + } + op := gcloud.Runf(t, "container clusters describe %s --zone %s --project %s", clusterName, location, projectId) + g := golden.NewOrUpdate(t, op.String(), + golden.WithSanitizer(golden.StringSanitizer(serviceAccount, "SERVICE_ACCOUNT")), + golden.WithSanitizer(golden.StringSanitizer(projectId, "PROJECT_ID")), + golden.WithSanitizer(golden.StringSanitizer(clusterName, "CLUSTER_NAME")), + ) + validateJSONPaths := []string{ + "autopilot.enabled", + "location", + "privateClusterConfig.enablePrivateEndpoint", + "privateClusterConfig.enablePrivateNodes", + "privateClusterConfig.addClusterFirewallRules", + "privateClusterConfig.addMasterWebhookFirewallRules", + "privateClusterConfig.addShadowFirewallRules", + } + + for _, pth := range validateJSONPaths { + g.JSONEq(assert, op, pth) + } + + assert.Contains([]string{"RUNNING", "RECONCILING"}, op.Get("status").String()) // comes up healthy + assert.Contains(op.Get("nodePoolAutoConfig.networkTags.tags").String(), "allow-google-apis") // example network_tag attached + assert.Contains(op.Get("nodePoolAutoConfig.networkTags.tags").String(), clusterNetworkTag) // the cluster_network_tag attached + + for _, n := range fws { + fw := gcloud.Runf(t, "compute firewall-rules --project %s describe %s", projectId, n) + assert.Contains(fw.Get("targetTags").String(), clusterNetworkTag) // firewall target tag is the cluster_network_tag + } + }) + bpt.Test() +} diff --git a/test/integration/autopilot_private_firewalls/testdata/TestAutopilotPrivateFirewalls.json b/test/integration/autopilot_private_firewalls/testdata/TestAutopilotPrivateFirewalls.json new file mode 100644 index 000000000..f0ebc6d6b --- /dev/null +++ b/test/integration/autopilot_private_firewalls/testdata/TestAutopilotPrivateFirewalls.json @@ -0,0 +1,651 @@ +{ + "addonsConfig": { + "dnsCacheConfig": { + "enabled": true + }, + "gcePersistentDiskCsiDriverConfig": { + "enabled": true + }, + "gcpFilestoreCsiDriverConfig": { + "enabled": true + }, + "horizontalPodAutoscaling": {}, + "httpLoadBalancing": {}, + "kubernetesDashboard": { + "disabled": true + }, + "networkPolicyConfig": { + "disabled": true + } + }, + "autopilot": { + "enabled": true + }, + "autoscaling": { + "autoprovisioningNodePoolDefaults": { + "imageType": "COS_CONTAINERD", + "management": { + "autoRepair": true, + "autoUpgrade": true + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/userinfo.email", + "https://www.googleapis.com/auth/cloud-platform" + ], + "serviceAccount": "SERVICE_ACCOUNT", + "upgradeSettings": { + "maxSurge": 1, + "strategy": "SURGE" + } + }, + "autoscalingProfile": "OPTIMIZE_UTILIZATION", + "enableNodeAutoprovisioning": true, + "resourceLimits": [ + { + "maximum": "1000000000", + "resourceType": "cpu" + }, + { + "maximum": "1000000000", + "resourceType": "memory" + }, + { + "maximum": "1000000000", + "resourceType": "nvidia-tesla-t4" + }, + { + "maximum": "1000000000", + "resourceType": "nvidia-tesla-a100" + } + ] + }, + "binaryAuthorization": {}, + "clusterIpv4Cidr": "192.168.0.0/18", + "createTime": "2023-05-31T03:55:39+00:00", + "currentMasterVersion": "1.25.8-gke.500", + "currentNodeCount": 2, + "currentNodeVersion": "1.25.8-gke.500", + "databaseEncryption": { + "state": "DECRYPTED" + }, + "defaultMaxPodsConstraint": { + "maxPodsPerNode": "110" + }, + "endpoint": "172.16.0.2", + "etag": "0a9c3c48-c0b7-4d97-adc4-32c2d5932e77", + "id": "7fef9574a79a47b58296ba7cbd693fbe1def4ffc85ce44cfa460136ebaba8662", + "initialClusterVersion": "1.25.8-gke.500", + "instanceGroupUrls": [ + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-f/instanceGroupManagers/gk3-autopilot-private-fi-default-pool-7732941f-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-c/instanceGroupManagers/gk3-autopilot-private-fi-default-pool-c1061d21-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-a/instanceGroupManagers/gk3-autopilot-private-firewall-pool-1-4cfe33eb-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-c/instanceGroupManagers/gk3-autopilot-private-firewall-pool-1-9fe99a0d-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-f/instanceGroupManagers/gk3-autopilot-private-firewall-pool-1-05734eb6-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-a/instanceGroupManagers/gk3-autopilot-private-firewall-pool-2-cf415b1a-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-c/instanceGroupManagers/gk3-autopilot-private-firewall-pool-2-56509f37-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-f/instanceGroupManagers/gk3-autopilot-private-firewall-pool-2-8f59aa08-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-a/instanceGroupManagers/gk3-autopilot-private-firewall-pool-3-779199f3-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-c/instanceGroupManagers/gk3-autopilot-private-firewall-pool-3-b6b392e2-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-f/instanceGroupManagers/gk3-autopilot-private-firewall-pool-3-d24d5320-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-a/instanceGroupManagers/gk3-autopilot-private-firewall-pool-4-198236ad-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-c/instanceGroupManagers/gk3-autopilot-private-firewall-pool-4-7aa8bf57-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-f/instanceGroupManagers/gk3-autopilot-private-firewall-pool-4-0c0020e0-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-a/instanceGroupManagers/gk3-autopilot-private-firewall-pool-5-2d5a875e-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-c/instanceGroupManagers/gk3-autopilot-private-firewall-pool-5-f44f3ecf-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-f/instanceGroupManagers/gk3-autopilot-private-firewall-pool-5-042d3615-grp" + ], + "ipAllocationPolicy": { + "additionalPodRangesConfig": {}, + "clusterIpv4Cidr": "192.168.0.0/18", + "clusterIpv4CidrBlock": "192.168.0.0/18", + "clusterSecondaryRangeName": "ip-range-pods-autopilot-private-firewalls", + "podCidrOverprovisionConfig": {}, + "servicesIpv4Cidr": "192.168.64.0/18", + "servicesIpv4CidrBlock": "192.168.64.0/18", + "servicesSecondaryRangeName": "ip-range-svc-autopilot-private-firewalls", + "stackType": "IPV4", + "useIpAliases": true + }, + "labelFingerprint": "a9dc16a7", + "legacyAbac": {}, + "location": "us-central1", + "locations": [ + "us-central1-a", + "us-central1-c", + "us-central1-f" + ], + "loggingConfig": { + "componentConfig": { + "enableComponents": [ + "SYSTEM_COMPONENTS", + "WORKLOADS" + ] + } + }, + "loggingService": "logging.googleapis.com/kubernetes", + "maintenancePolicy": { + "resourceVersion": "ce912209", + "window": { + "dailyMaintenanceWindow": { + "duration": "PT4H0M0S", + "startTime": "05:00" + } + } + }, + "masterAuth": { + "clusterCaCertificate": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVMRENDQXBTZ0F3SUJBZ0lRZGg2ZnJnM21EOFpSMFNmREt2TmtwakFOQmdrcWhraUc5dzBCQVFzRkFEQXYKTVMwd0t3WURWUVFERXlSaE16ZGtZMkl5WmkxbVpUZzRMVFJrWlRBdE9EZGlNaTA1WkRReE5EVXdNVGhpT0RBdwpJQmNOTWpNd05UTXhNREkxTlRRd1doZ1BNakExTXpBMU1qTXdNelUxTkRCYU1DOHhMVEFyQmdOVkJBTVRKR0V6Ck4yUmpZakptTFdabE9EZ3ROR1JsTUMwNE4ySXlMVGxrTkRFME5UQXhPR0k0TURDQ0FhSXdEUVlKS29aSWh2Y04KQVFFQkJRQURnZ0dQQURDQ0FZb0NnZ0dCQUtZODdIY25YaWdvbTFEVnZrUG1aQ1h2VXk2RXFndjI3eHcxelJEWgp2OGNYUWYwenQyNVViRTRkZWlIY3lVcHYwTGhVL3FYaVErYkt4SUlJdkRYL2J1NzFyaGxHcExUb3d5b0RHTTh6ClUrYVQvYjZlcmVtd1VxcSs3cmNXZXdDM3NCU0ZOTzdEWk1VOHpKL0docFVyWmowY0pTNnRxVHFOeW1wbU0ySnoKaTlobTRXekEzeVhzNGJRSjQxV1I1SXFWbFFncTlsM0J5ZWpVaUM4RS9rTEk5eEl0Tll5QVArOTVMRmFnek9GNApFdzZwZU55d3B4YVFTcXVJYWx0MUx3bW05bWVRL2RsZTdxczFmRTIvbjhEWmkzMFlxY1hESXFacHhRaUFjOFgxCkRIUlIxbkNJZWkxV2NRcXd6L2NMRHlkTGlneUdmU0ZpRzAyK1hhNHQwL011TmN2Z2sxVXQydEtnejh5WVpmNGkKMlBJcHBRYlVvNTc5OXpoQm5MRkNjVVhoTk5BV1lCMERIZjlVK05FYTVVMmJsbGtpYVA0YnAvN24yNXRUQlZXRApxdkhkd0czalVwaThDNm94a0lhZ0dZZUxSVkhBL1dZZlJXUEtjcDgxVDdVOEgraGM3VnRVdCtDeXJ5dTI1dVpuCkp1Z2pGVWhzKzJMayswRnBYT3VWWTRZaS9RSURBUUFCbzBJd1FEQU9CZ05WSFE4QkFmOEVCQU1DQWdRd0R3WUQKVlIwVEFRSC9CQVV3QXdFQi96QWRCZ05WSFE0RUZnUVVEeHc4eldmakJIM3dIVHd3bDJIZTdYV0lkTjR3RFFZSgpLb1pJaHZjTkFRRUxCUUFEZ2dHQkFJWEVoNnczT0lINWJhZDROaEwyNFlFV3pLU2lDUWdSN1ZxTk9GVWl2RGdVCnpwVGNiYnp5QmluZ00zRGNMTGlzTFByOUlJUkpjNGE4MmpIcHQyV3NOL1BvYjZXcTlSelA4UE8zSWx5OUZyVFgKVmIxdUllT3dTdXBlZVVQYkRaMUdHUzlRdXVIT1ExaXFzcEgyZkUyVk9aS0c1eXVFRHJFSmllRUk4NFQ2UU8xZgpSMXlDd1FuaTNEMHlpL2R6MTBOck9iK1crclAxV2FlbnIzQUlzYS9KbUxwUlRTYVc2QnExaUdrQkxWYmZwaEx3Ck1IV2FlRlZQc2hCOUZNeG9DNUkxMVd2VUpIazQzV29SY2J3c2RJSjJ4a1RsZHVVaFF6RDFManFjd1gvenVYUnIKaU1IclhMWjQ3NzJiWGhzeHdFKzJDY3JDRzZ3ZFFZbTlrbXJIaksyeStCM1FWYTBXUXYyWGVGMERSWnFWZGxMNAp0MGhWcVNQbUdyRHArNFgrbWhmSEh5OXZ6T1IxM0FFekJTYzcxVUVMVFkvR0krcnVKWnFLQmJFd1dhS3F5ZE85CmJtaGZxWnY0bmFIbGswdDRiZlpOVHoxdUxpRmdEc1k3UlNFQWY4cXUzYVppaElSVEUxLy9LR3U1ZFlYa1pzNGEKRmh4UEJXSHR2a3Y4bjhlM2QyQnczQT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K" + }, + "masterAuthorizedNetworksConfig": { + "cidrBlocks": [ + { + "cidrBlock": "10.60.0.0/17", + "displayName": "VPC" + } + ], + "enabled": true + }, + "monitoringConfig": { + "componentConfig": { + "enableComponents": [ + "SYSTEM_COMPONENTS" + ] + }, + "managedPrometheusConfig": { + "enabled": true + } + }, + "monitoringService": "monitoring.googleapis.com/kubernetes", + "name": "CLUSTER_NAME", + "network": "autopilot-private-firewalls-network", + "networkConfig": { + "datapathProvider": "ADVANCED_DATAPATH", + "defaultSnatStatus": {}, + "enableIntraNodeVisibility": true, + "network": "projects/PROJECT_ID/global/networks/autopilot-private-firewalls-network", + "serviceExternalIpsConfig": {}, + "subnetwork": "projects/PROJECT_ID/regions/us-central1/subnetworks/autopilot-private-firewalls-subnet" + }, + "nodeConfig": { + "diskSizeGb": 100, + "diskType": "pd-standard", + "imageType": "COS_CONTAINERD", + "machineType": "e2-medium", + "metadata": { + "disable-legacy-endpoints": "true" + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/userinfo.email", + "https://www.googleapis.com/auth/cloud-platform" + ], + "reservationAffinity": { + "consumeReservationType": "NO_RESERVATION" + }, + "serviceAccount": "SERVICE_ACCOUNT", + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true, + "enableSecureBoot": true + }, + "tags": [ + "allow-google-apis", + "gke-CLUSTER_NAME" + ], + "windowsNodeConfig": {}, + "workloadMetadataConfig": { + "mode": "GKE_METADATA" + } + }, + "nodePoolAutoConfig": { + "networkTags": { + "tags": [ + "allow-google-apis", + "gke-CLUSTER_NAME" + ] + } + }, + "nodePoolDefaults": { + "nodeConfigDefaults": { + "gcfsConfig": { + "enabled": true + }, + "loggingConfig": { + "variantConfig": { + "variant": "DEFAULT" + } + } + } + }, + "nodePools": [ + { + "autoscaling": { + "enabled": true, + "locationPolicy": "BALANCED", + "maxNodeCount": 1000 + }, + "config": { + "diskSizeGb": 100, + "diskType": "pd-standard", + "imageType": "COS_CONTAINERD", + "machineType": "e2-medium", + "metadata": { + "disable-legacy-endpoints": "true" + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/userinfo.email", + "https://www.googleapis.com/auth/cloud-platform" + ], + "reservationAffinity": { + "consumeReservationType": "NO_RESERVATION" + }, + "serviceAccount": "SERVICE_ACCOUNT", + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true, + "enableSecureBoot": true + }, + "tags": [ + "allow-google-apis", + "gke-CLUSTER_NAME" + ], + "windowsNodeConfig": {}, + "workloadMetadataConfig": { + "mode": "GKE_METADATA" + } + }, + "etag": "45c7e170-8f72-410c-81cb-d78a369e4866", + "initialNodeCount": 1, + "instanceGroupUrls": [ + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-f/instanceGroupManagers/gk3-autopilot-private-fi-default-pool-7732941f-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-c/instanceGroupManagers/gk3-autopilot-private-fi-default-pool-c1061d21-grp" + ], + "locations": [ + "us-central1-f", + "us-central1-c" + ], + "management": { + "autoRepair": true, + "autoUpgrade": true + }, + "maxPodsConstraint": { + "maxPodsPerNode": "32" + }, + "name": "default-pool", + "networkConfig": { + "podIpv4CidrBlock": "192.168.0.0/18", + "podRange": "ip-range-pods-autopilot-private-firewalls" + }, + "podIpv4CidrSize": 26, + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/us-central1/clusters/CLUSTER_NAME/nodePools/default-pool", + "status": "RUNNING", + "upgradeSettings": { + "maxSurge": 1, + "strategy": "SURGE" + }, + "version": "1.25.8-gke.500" + }, + { + "autoscaling": { + "enabled": true, + "locationPolicy": "BALANCED", + "maxNodeCount": 1000 + }, + "config": { + "diskSizeGb": 100, + "diskType": "pd-standard", + "imageType": "COS_CONTAINERD", + "machineType": "e2-standard-2", + "metadata": { + "disable-legacy-endpoints": "true" + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/userinfo.email", + "https://www.googleapis.com/auth/cloud-platform" + ], + "reservationAffinity": { + "consumeReservationType": "NO_RESERVATION" + }, + "serviceAccount": "SERVICE_ACCOUNT", + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true, + "enableSecureBoot": true + }, + "tags": [ + "allow-google-apis", + "gke-CLUSTER_NAME" + ], + "windowsNodeConfig": {}, + "workloadMetadataConfig": { + "mode": "GKE_METADATA" + } + }, + "etag": "6d016d3f-178f-4f4d-9b59-ced698a74fc4", + "instanceGroupUrls": [ + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-a/instanceGroupManagers/gk3-autopilot-private-firewall-pool-1-4cfe33eb-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-c/instanceGroupManagers/gk3-autopilot-private-firewall-pool-1-9fe99a0d-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-f/instanceGroupManagers/gk3-autopilot-private-firewall-pool-1-05734eb6-grp" + ], + "locations": [ + "us-central1-a", + "us-central1-c", + "us-central1-f" + ], + "management": { + "autoRepair": true, + "autoUpgrade": true + }, + "maxPodsConstraint": { + "maxPodsPerNode": "32" + }, + "name": "pool-1", + "networkConfig": { + "podIpv4CidrBlock": "192.168.0.0/18", + "podRange": "ip-range-pods-autopilot-private-firewalls" + }, + "podIpv4CidrSize": 26, + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/us-central1/clusters/CLUSTER_NAME/nodePools/pool-1", + "status": "RUNNING", + "upgradeSettings": { + "maxSurge": 1, + "strategy": "SURGE" + }, + "version": "1.25.8-gke.500" + }, + { + "autoscaling": { + "enabled": true, + "locationPolicy": "BALANCED", + "maxNodeCount": 1000 + }, + "config": { + "diskSizeGb": 100, + "diskType": "pd-standard", + "imageType": "COS_CONTAINERD", + "machineType": "e2-standard-4", + "metadata": { + "disable-legacy-endpoints": "true" + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/userinfo.email", + "https://www.googleapis.com/auth/cloud-platform" + ], + "reservationAffinity": { + "consumeReservationType": "NO_RESERVATION" + }, + "serviceAccount": "SERVICE_ACCOUNT", + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true, + "enableSecureBoot": true + }, + "tags": [ + "allow-google-apis", + "gke-CLUSTER_NAME" + ], + "windowsNodeConfig": {}, + "workloadMetadataConfig": { + "mode": "GKE_METADATA" + } + }, + "etag": "95e67a5b-95b5-419b-9d9b-0dac3bd9602a", + "instanceGroupUrls": [ + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-a/instanceGroupManagers/gk3-autopilot-private-firewall-pool-2-cf415b1a-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-c/instanceGroupManagers/gk3-autopilot-private-firewall-pool-2-56509f37-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-f/instanceGroupManagers/gk3-autopilot-private-firewall-pool-2-8f59aa08-grp" + ], + "locations": [ + "us-central1-a", + "us-central1-c", + "us-central1-f" + ], + "management": { + "autoRepair": true, + "autoUpgrade": true + }, + "maxPodsConstraint": { + "maxPodsPerNode": "32" + }, + "name": "pool-2", + "networkConfig": { + "podIpv4CidrBlock": "192.168.0.0/18", + "podRange": "ip-range-pods-autopilot-private-firewalls" + }, + "podIpv4CidrSize": 26, + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/us-central1/clusters/CLUSTER_NAME/nodePools/pool-2", + "status": "RUNNING", + "upgradeSettings": { + "maxSurge": 1, + "strategy": "SURGE" + }, + "version": "1.25.8-gke.500" + }, + { + "autoscaling": { + "enabled": true, + "locationPolicy": "BALANCED", + "maxNodeCount": 1000 + }, + "config": { + "diskSizeGb": 100, + "diskType": "pd-standard", + "imageType": "COS_CONTAINERD", + "machineType": "e2-standard-8", + "metadata": { + "disable-legacy-endpoints": "true" + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/userinfo.email", + "https://www.googleapis.com/auth/cloud-platform" + ], + "reservationAffinity": { + "consumeReservationType": "NO_RESERVATION" + }, + "serviceAccount": "SERVICE_ACCOUNT", + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true, + "enableSecureBoot": true + }, + "tags": [ + "allow-google-apis", + "gke-CLUSTER_NAME" + ], + "windowsNodeConfig": {}, + "workloadMetadataConfig": { + "mode": "GKE_METADATA" + } + }, + "etag": "f4814faf-6986-48d4-b913-3f85af33dfae", + "instanceGroupUrls": [ + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-a/instanceGroupManagers/gk3-autopilot-private-firewall-pool-3-779199f3-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-c/instanceGroupManagers/gk3-autopilot-private-firewall-pool-3-b6b392e2-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-f/instanceGroupManagers/gk3-autopilot-private-firewall-pool-3-d24d5320-grp" + ], + "locations": [ + "us-central1-a", + "us-central1-c", + "us-central1-f" + ], + "management": { + "autoRepair": true, + "autoUpgrade": true + }, + "maxPodsConstraint": { + "maxPodsPerNode": "32" + }, + "name": "pool-3", + "networkConfig": { + "podIpv4CidrBlock": "192.168.0.0/18", + "podRange": "ip-range-pods-autopilot-private-firewalls" + }, + "podIpv4CidrSize": 26, + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/us-central1/clusters/CLUSTER_NAME/nodePools/pool-3", + "status": "RUNNING", + "upgradeSettings": { + "maxSurge": 1, + "strategy": "SURGE" + }, + "version": "1.25.8-gke.500" + }, + { + "autoscaling": { + "enabled": true, + "locationPolicy": "BALANCED", + "maxNodeCount": 1000 + }, + "config": { + "diskSizeGb": 100, + "diskType": "pd-standard", + "imageType": "COS_CONTAINERD", + "machineType": "e2-standard-16", + "metadata": { + "disable-legacy-endpoints": "true" + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/userinfo.email", + "https://www.googleapis.com/auth/cloud-platform" + ], + "reservationAffinity": { + "consumeReservationType": "NO_RESERVATION" + }, + "serviceAccount": "SERVICE_ACCOUNT", + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true, + "enableSecureBoot": true + }, + "tags": [ + "allow-google-apis", + "gke-CLUSTER_NAME" + ], + "windowsNodeConfig": {}, + "workloadMetadataConfig": { + "mode": "GKE_METADATA" + } + }, + "etag": "0921ac09-d6fe-47a1-9897-6d3c3fe113a1", + "instanceGroupUrls": [ + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-a/instanceGroupManagers/gk3-autopilot-private-firewall-pool-4-198236ad-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-c/instanceGroupManagers/gk3-autopilot-private-firewall-pool-4-7aa8bf57-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-f/instanceGroupManagers/gk3-autopilot-private-firewall-pool-4-0c0020e0-grp" + ], + "locations": [ + "us-central1-a", + "us-central1-c", + "us-central1-f" + ], + "management": { + "autoRepair": true, + "autoUpgrade": true + }, + "maxPodsConstraint": { + "maxPodsPerNode": "32" + }, + "name": "pool-4", + "networkConfig": { + "podIpv4CidrBlock": "192.168.0.0/18", + "podRange": "ip-range-pods-autopilot-private-firewalls" + }, + "podIpv4CidrSize": 26, + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/us-central1/clusters/CLUSTER_NAME/nodePools/pool-4", + "status": "RUNNING", + "upgradeSettings": { + "maxSurge": 1, + "strategy": "SURGE" + }, + "version": "1.25.8-gke.500" + }, + { + "autoscaling": { + "enabled": true, + "locationPolicy": "BALANCED", + "maxNodeCount": 1000 + }, + "config": { + "diskSizeGb": 100, + "diskType": "pd-standard", + "imageType": "COS_CONTAINERD", + "machineType": "e2-standard-32", + "metadata": { + "disable-legacy-endpoints": "true" + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/userinfo.email", + "https://www.googleapis.com/auth/cloud-platform" + ], + "reservationAffinity": { + "consumeReservationType": "NO_RESERVATION" + }, + "serviceAccount": "SERVICE_ACCOUNT", + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true, + "enableSecureBoot": true + }, + "tags": [ + "allow-google-apis", + "gke-CLUSTER_NAME" + ], + "windowsNodeConfig": {}, + "workloadMetadataConfig": { + "mode": "GKE_METADATA" + } + }, + "etag": "1d141704-04fc-4f24-9da1-b6105f7309c5", + "instanceGroupUrls": [ + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-a/instanceGroupManagers/gk3-autopilot-private-firewall-pool-5-2d5a875e-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-c/instanceGroupManagers/gk3-autopilot-private-firewall-pool-5-f44f3ecf-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-central1-f/instanceGroupManagers/gk3-autopilot-private-firewall-pool-5-042d3615-grp" + ], + "locations": [ + "us-central1-a", + "us-central1-c", + "us-central1-f" + ], + "management": { + "autoRepair": true, + "autoUpgrade": true + }, + "maxPodsConstraint": { + "maxPodsPerNode": "32" + }, + "name": "pool-5", + "networkConfig": { + "podIpv4CidrBlock": "192.168.0.0/18", + "podRange": "ip-range-pods-autopilot-private-firewalls" + }, + "podIpv4CidrSize": 26, + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/us-central1/clusters/CLUSTER_NAME/nodePools/pool-5", + "status": "RUNNING", + "upgradeSettings": { + "maxSurge": 1, + "strategy": "SURGE" + }, + "version": "1.25.8-gke.500" + } + ], + "notificationConfig": { + "pubsub": {} + }, + "privateClusterConfig": { + "enablePrivateEndpoint": true, + "enablePrivateNodes": true, + "masterGlobalAccessConfig": { + "enabled": true + }, + "masterIpv4CidrBlock": "172.16.0.0/28", + "peeringName": "gke-n425cd81899d813812bb-278b-fee3-peer", + "privateEndpoint": "172.16.0.2", + "publicEndpoint": "35.192.78.37" + }, + "releaseChannel": { + "channel": "REGULAR" + }, + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/us-central1/clusters/CLUSTER_NAME", + "servicesIpv4Cidr": "192.168.64.0/18", + "shieldedNodes": { + "enabled": true + }, + "status": "RUNNING", + "subnetwork": "autopilot-private-firewalls-subnet", + "verticalPodAutoscaling": { + "enabled": true + }, + "workloadIdentityConfig": { + "workloadPool": "PROJECT_ID.svc.id.goog" + }, + "zone": "us-central1" +}