From fe0ff7ae462db4e8541ac2c9bf3cd41140e9ca80 Mon Sep 17 00:00:00 2001 From: Francis Liu Date: Thu, 27 Oct 2022 23:04:43 +0000 Subject: [PATCH 01/11] Add support for container cluster resource for gcp_public_cidrs_access_enabled Signed-off-by: Francis Liu --- .../resource_container_cluster.go.erb | 28 +++++-- .../resource_container_cluster_test.go.erb | 73 +++++++++++++++++++ .../docs/r/container_cluster.html.markdown | 3 + 3 files changed, 96 insertions(+), 8 deletions(-) diff --git a/mmv1/third_party/terraform/resources/resource_container_cluster.go.erb b/mmv1/third_party/terraform/resources/resource_container_cluster.go.erb index 9ec8821bc8da..27eb987d9be7 100644 --- a/mmv1/third_party/terraform/resources/resource_container_cluster.go.erb +++ b/mmv1/third_party/terraform/resources/resource_container_cluster.go.erb @@ -28,17 +28,23 @@ import ( var ( instanceGroupManagerURL = regexp.MustCompile(fmt.Sprintf("projects/(%s)/zones/([a-z0-9-]*)/instanceGroupManagers/([^/]*)", ProjectRegex)) - networkConfig = &schema.Resource{ + masterAuthorizedNetworksConfig = &schema.Resource{ Schema: map[string]*schema.Schema{ "cidr_blocks": { Type: schema.TypeSet, - // Despite being the only entry in a nested block, this should be kept - // Optional. Expressing the parent with no entries and omitting the + // This should be kept Optional. Expressing the + // parent with no entries and omitting the // parent entirely are semantically different. Optional: true, Elem: cidrBlockConfig, Description: `External networks that can access the Kubernetes cluster master through HTTPS.`, }, + "gcp_public_cidrs_access_enabled": { + Type: schema.TypeBool, + Optional: true, + Computed: true, + Description: `Whether master is accessbile via Google Compute Engine Public IP addresses.`, + }, }, } cidrBlockConfig = &schema.Resource{ @@ -1125,8 +1131,9 @@ func resourceContainerCluster() *schema.Resource { "master_authorized_networks_config": { Type: schema.TypeList, Optional: true, + Computed: true, MaxItems: 1, - Elem: networkConfig, + Elem: masterAuthorizedNetworksConfig, Description: `The desired configuration options for master authorized networks. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists).`, }, @@ -1835,7 +1842,7 @@ func resourceContainerClusterCreate(d *schema.ResourceData, meta interface{}) er Name: clusterName, InitialNodeCount: int64(d.Get("initial_node_count").(int)), MaintenancePolicy: expandMaintenancePolicy(d, meta), - MasterAuthorizedNetworksConfig: expandMasterAuthorizedNetworksConfig(d.Get("master_authorized_networks_config")), + MasterAuthorizedNetworksConfig: expandMasterAuthorizedNetworksConfig(d.Get("master_authorized_networks_config"), d), InitialClusterVersion: d.Get("min_master_version").(string), ClusterIpv4Cidr: d.Get("cluster_ipv4_cidr").(string), Description: d.Get("description").(string), @@ -2453,7 +2460,7 @@ func resourceContainerClusterUpdate(d *schema.ResourceData, meta interface{}) er c := d.Get("master_authorized_networks_config") req := &container.UpdateClusterRequest{ Update: &container.ClusterUpdate{ - DesiredMasterAuthorizedNetworksConfig: expandMasterAuthorizedNetworksConfig(c), + DesiredMasterAuthorizedNetworksConfig: expandMasterAuthorizedNetworksConfig(c, d), }, } @@ -4061,7 +4068,7 @@ func expandMasterAuth(configured interface{}) *container.MasterAuth { return result } -func expandMasterAuthorizedNetworksConfig(configured interface{}) *container.MasterAuthorizedNetworksConfig { +func expandMasterAuthorizedNetworksConfig(configured interface{}, d *schema.ResourceData) *container.MasterAuthorizedNetworksConfig { l := configured.([]interface{}) if len(l) == 0 { return &container.MasterAuthorizedNetworksConfig{ @@ -4083,6 +4090,10 @@ func expandMasterAuthorizedNetworksConfig(configured interface{}) *container.Mas }) } } + if v, ok := d.GetOkExists("master_authorized_networks_config.0.gcp_public_cidrs_access_enabled"); ok { + result.GcpPublicCidrsAccessEnabled = v.(bool) + result.ForceSendFields = []string{"GcpPublicCidrsAccessEnabled"} + } } return result } @@ -4949,7 +4960,7 @@ func flattenStandardRolloutPolicy(a *container.StandardRolloutPolicy) []map[stri } func flattenMasterAuthorizedNetworksConfig(c *container.MasterAuthorizedNetworksConfig) []map[string]interface{} { - if c == nil || !c.Enabled { + if c == nil { return nil } result := make(map[string]interface{}) @@ -4963,6 +4974,7 @@ func flattenMasterAuthorizedNetworksConfig(c *container.MasterAuthorizedNetworks } result["cidr_blocks"] = schema.NewSet(schema.HashResource(cidrBlockConfig), cidrBlocks) } + result["gcp_public_cidrs_access_enabled"] = c.GcpPublicCidrsAccessEnabled return []map[string]interface{}{result} } diff --git a/mmv1/third_party/terraform/tests/resource_container_cluster_test.go.erb b/mmv1/third_party/terraform/tests/resource_container_cluster_test.go.erb index 44d153cc06ca..0ff5821cb636 100644 --- a/mmv1/third_party/terraform/tests/resource_container_cluster_test.go.erb +++ b/mmv1/third_party/terraform/tests/resource_container_cluster_test.go.erb @@ -714,6 +714,55 @@ func TestAccContainerCluster_withMasterAuthorizedNetworksConfig(t *testing.T) { }) } +func TestAccContainerCluster_withGcpPublicCidrsAccessEnabledToggle(t *testing.T) { + t.Parallel() + + clusterName := fmt.Sprintf("tf-test-cluster-%s", randString(t, 10)) + + vcrTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckContainerClusterDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccContainerCluster_withGcpPublicCidrsAccessEnabled(clusterName, "", ""), + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("google_container_cluster.with_gcp_public_cidrs_access_enabled", + "master_authorized_networks_config.#", "1"), + resource.TestCheckResourceAttr("google_container_cluster.with_gcp_public_cidrs_access_enabled", + "master_authorized_networks_config.0.gcp_public_cidrs_access_enabled", "true"), + ), + }, + { + ResourceName: "google_container_cluster.with_gcp_public_cidrs_access_enabled", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"min_master_version"}, + }, + { + Config: testAccContainerCluster_withGcpPublicCidrsAccessEnabled(clusterName, "false", ""), + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("google_container_cluster.with_gcp_public_cidrs_access_enabled", + "master_authorized_networks_config.0.gcp_public_cidrs_access_enabled", "false"), + ), + }, + { + ResourceName: "google_container_cluster.with_gcp_public_cidrs_access_enabled", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"min_master_version"}, + }, + { + Config: testAccContainerCluster_withGcpPublicCidrsAccessEnabled(clusterName, "true", ""), + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("google_container_cluster.with_gcp_public_cidrs_access_enabled", + "master_authorized_networks_config.0.gcp_public_cidrs_access_enabled", "true"), + ), + }, + }, + }) +} + func TestAccContainerCluster_regional(t *testing.T) { t.Parallel() @@ -4000,6 +4049,30 @@ resource "google_container_cluster" "with_master_authorized_networks" { `, clusterName, cidrBlocks) } +func testAccContainerCluster_withGcpPublicCidrsAccessEnabled(clusterName string, flag string, emptyValue string) string { + + v := emptyValue + if flag != emptyValue { + var buf bytes.Buffer + buf.WriteString(fmt.Sprintf(` + master_authorized_networks_config { + gcp_public_cidrs_access_enabled = %s + }`, flag)) + v = buf.String() + } + + return fmt.Sprintf(` +resource "google_container_cluster" "with_gcp_public_cidrs_access_enabled" { + name = "%s" + location = "us-central1-a" + min_master_version = "1.23" + initial_node_count = 1 + + %s +} +`, clusterName, v) +} + func testAccContainerCluster_removeMasterAuthorizedNetworksConfig(clusterName string) string { return fmt.Sprintf(` resource "google_container_cluster" "with_master_authorized_networks" { diff --git a/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown b/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown index 269c93e19402..1af0c0e85114 100644 --- a/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown +++ b/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown @@ -696,6 +696,9 @@ This block also contains several computed attributes, documented below. * `cidr_blocks` - (Optional) External networks that can access the Kubernetes cluster master through HTTPS. +* `with_gcp_public_cidrs_access_enabled` - (Optional) Whether Kubernetes master is + accessible via Google Compute Engine Public IPs. + The `master_authorized_networks_config.cidr_blocks` block supports: * `cidr_block` - (Optional) External network that can access Kubernetes master through HTTPS. From 4a3a334ecc0659679219464c48c7eacf6f082785 Mon Sep 17 00:00:00 2001 From: Francis Liu Date: Mon, 31 Oct 2022 16:47:01 +0000 Subject: [PATCH 02/11] Add support for container cluster resource for private_cluster_config.0.private_endpoint_subnetwork Signed-off-by: Francis Liu --- .../resource_container_cluster.go.erb | 30 +++++--- .../resource_container_cluster_test.go.erb | 71 +++++++++++++++++++ .../docs/r/container_cluster.html.markdown | 2 + 3 files changed, 94 insertions(+), 9 deletions(-) diff --git a/mmv1/third_party/terraform/resources/resource_container_cluster.go.erb b/mmv1/third_party/terraform/resources/resource_container_cluster.go.erb index 27eb987d9be7..fa44152480b5 100644 --- a/mmv1/third_party/terraform/resources/resource_container_cluster.go.erb +++ b/mmv1/third_party/terraform/resources/resource_container_cluster.go.erb @@ -1382,7 +1382,7 @@ func resourceContainerCluster() *schema.Resource { Schema: map[string]*schema.Schema{ "enable_private_endpoint": { Type: schema.TypeBool, - Required: true, + Optional: true, ForceNew: true, DiffSuppressFunc: containerClusterPrivateClusterConfigSuppress, Description: `When true, the cluster's private endpoint is used as the cluster endpoint and access through the public endpoint is disabled. When false, either endpoint can be used. This field only applies to private clusters, when enable_private_nodes is true.`, @@ -1412,6 +1412,13 @@ func resourceContainerCluster() *schema.Resource { Computed: true, Description: `The internal IP address of this cluster's master endpoint.`, }, + "private_endpoint_subnetwork": { + Type: schema.TypeString, + Optional: true, + ForceNew: true, + DiffSuppressFunc: compareSelfLinkOrResourceName, + Description: `Subnetwork in cluster's network where master's endpoint will be provisioned.`, + }, "public_endpoint": { Type: schema.TypeString, Computed: true, @@ -4121,11 +4128,12 @@ func expandPrivateClusterConfig(configured interface{}) *container.PrivateCluste } config := l[0].(map[string]interface{}) return &container.PrivateClusterConfig{ - EnablePrivateEndpoint: config["enable_private_endpoint"].(bool), - EnablePrivateNodes: config["enable_private_nodes"].(bool), - MasterIpv4CidrBlock: config["master_ipv4_cidr_block"].(string), - MasterGlobalAccessConfig: expandPrivateClusterConfigMasterGlobalAccessConfig(config["master_global_access_config"]), - ForceSendFields: []string{"EnablePrivateEndpoint", "EnablePrivateNodes", "MasterIpv4CidrBlock", "MasterGlobalAccessConfig"}, + EnablePrivateEndpoint: config["enable_private_endpoint"].(bool), + EnablePrivateNodes: config["enable_private_nodes"].(bool), + MasterIpv4CidrBlock: config["master_ipv4_cidr_block"].(string), + MasterGlobalAccessConfig: expandPrivateClusterConfigMasterGlobalAccessConfig(config["master_global_access_config"]), + PrivateEndpointSubnetwork: config["private_endpoint_subnetwork"].(string), + ForceSendFields: []string{"EnablePrivateEndpoint", "EnablePrivateNodes", "MasterIpv4CidrBlock", "MasterGlobalAccessConfig"}, } } @@ -4675,6 +4683,7 @@ func flattenPrivateClusterConfig(c *container.PrivateClusterConfig) []map[string "master_global_access_config": flattenPrivateClusterConfigMasterGlobalAccessConfig(c.MasterGlobalAccessConfig), "peering_name": c.PeeringName, "private_endpoint": c.PrivateEndpoint, + "private_endpoint_subnetwork": c.PrivateEndpointSubnetwork, "public_endpoint": c.PublicEndpoint, }, } @@ -5253,12 +5262,15 @@ func containerClusterPrivateClusterConfigSuppress(k, old, new string, d *schema. o, n = d.GetChange("private_cluster_config.0.enable_private_nodes") suppressNodes := !o.(bool) && !n.(bool) + // Do not suppress diffs when private_endpoint_subnetwork is configured + _, hasSubnet := d.GetOk("private_cluster_config.0.private_endpoint_subnetwork") + if k == "private_cluster_config.0.enable_private_endpoint" { - return suppressEndpoint + return suppressEndpoint && !hasSubnet } else if k == "private_cluster_config.0.enable_private_nodes" { - return suppressNodes + return suppressNodes && !hasSubnet } else if k == "private_cluster_config.#" { - return suppressEndpoint && suppressNodes + return suppressEndpoint && suppressNodes && !hasSubnet } return false } diff --git a/mmv1/third_party/terraform/tests/resource_container_cluster_test.go.erb b/mmv1/third_party/terraform/tests/resource_container_cluster_test.go.erb index 0ff5821cb636..8f081122de9c 100644 --- a/mmv1/third_party/terraform/tests/resource_container_cluster_test.go.erb +++ b/mmv1/third_party/terraform/tests/resource_container_cluster_test.go.erb @@ -4093,6 +4093,77 @@ resource "google_container_cluster" "regional" { `, clusterName) } +func TestAccContainerCluster_withPrivateEndpointSubnetwork(t *testing.T) { + t.Parallel() + + r := randString(t, 10) + + subnet1Name := fmt.Sprintf("tf-test-container-subnetwork1-%s", r) + subnet1Cidr := "10.0.36.0/24" + + subnet2Name := fmt.Sprintf("tf-test-container-subnetwork2-%s", r) + subnet2Cidr := "10.9.26.0/24" + + clusterName := fmt.Sprintf("tf-test-cluster-%s", randString(t, 10)) + containerNetName := fmt.Sprintf("tf-test-container-net-%s", r) + + vcrTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckContainerClusterDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccContainerCluster_withPrivateEndpointSubnetwork(containerNetName, clusterName, subnet1Name, subnet1Cidr, subnet2Name, subnet2Cidr), + }, + { + ResourceName: "google_container_cluster.with_private_endpoint_subnetwork", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"min_master_version"}, + }, + }, + }) +} + +func testAccContainerCluster_withPrivateEndpointSubnetwork(containerNetName, clusterName, s1Name, s1Cidr, s2Name, s2Cidr string) string { + return fmt.Sprintf(` +resource "google_compute_network" "container_network" { + name = "%s" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "container_subnetwork1" { + name = "%s" + network = google_compute_network.container_network.name + ip_cidr_range = "%s" + region = "us-central1" + private_ip_google_access = true +} + +resource "google_compute_subnetwork" "container_subnetwork2" { + name = "%s" + network = google_compute_network.container_network.name + ip_cidr_range = "%s" + region = "us-central1" + private_ip_google_access = true +} + +resource "google_container_cluster" "with_private_endpoint_subnetwork" { + name = "%s" + location = "us-central1-a" + min_master_version = "1.23" + initial_node_count = 1 + + network = google_compute_network.container_network.name + subnetwork = google_compute_subnetwork.container_subnetwork1.name + + private_cluster_config { + private_endpoint_subnetwork = google_compute_subnetwork.container_subnetwork2.name + } +} +`, containerNetName, s1Name, s1Cidr, s2Name, s2Cidr, clusterName) +} + func testAccContainerCluster_regionalWithNodePool(cluster, nodePool string) string { return fmt.Sprintf(` resource "google_container_cluster" "regional" { diff --git a/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown b/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown index 1af0c0e85114..3546c12814bf 100644 --- a/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown +++ b/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown @@ -989,6 +989,8 @@ In addition, the `private_cluster_config` allows access to the following read-on * `private_endpoint` - The internal IP address of this cluster's master endpoint. +* `private_endpoint_subnetwork` - Subnetwork in cluster's network where master's endpoint will be provisioned. + * `public_endpoint` - The external IP address of this cluster's master endpoint. !> The Google provider is unable to validate certain configurations of From 6d3299e6495676d4943330cc93bc8ed076e16bb2 Mon Sep 17 00:00:00 2001 From: Francis Liu Date: Mon, 31 Oct 2022 18:35:58 +0000 Subject: [PATCH 03/11] Add support for container cluster resource for mutable private_cluster_config.0.enable_private_endpoint Signed-off-by: Francis Liu --- .../resource_container_cluster.go.erb | 19 ++++- .../resource_container_cluster_test.go.erb | 69 +++++++++++++++++++ 2 files changed, 87 insertions(+), 1 deletion(-) diff --git a/mmv1/third_party/terraform/resources/resource_container_cluster.go.erb b/mmv1/third_party/terraform/resources/resource_container_cluster.go.erb index fa44152480b5..321b70dc64cf 100644 --- a/mmv1/third_party/terraform/resources/resource_container_cluster.go.erb +++ b/mmv1/third_party/terraform/resources/resource_container_cluster.go.erb @@ -1383,7 +1383,6 @@ func resourceContainerCluster() *schema.Resource { "enable_private_endpoint": { Type: schema.TypeBool, Optional: true, - ForceNew: true, DiffSuppressFunc: containerClusterPrivateClusterConfigSuppress, Description: `When true, the cluster's private endpoint is used as the cluster endpoint and access through the public endpoint is disabled. When false, either endpoint can be used. This field only applies to private clusters, when enable_private_nodes is true.`, }, @@ -2531,6 +2530,24 @@ func resourceContainerClusterUpdate(d *schema.ResourceData, meta interface{}) er log.Printf("[INFO] GKE cluster %s's binary authorization has been updated to %v", d.Id(), enabled) } + if d.HasChange("private_cluster_config.0.enable_private_endpoint") { + enabled := d.Get("private_cluster_config.0.enable_private_endpoint").(bool) + req := &container.UpdateClusterRequest{ + Update: &container.ClusterUpdate{ + DesiredEnablePrivateEndpoint: enabled, + ForceSendFields: []string{"DesiredEnablePrivateEndpoint"}, + }, + } + + updateF := updateFunc(req, "updating enable private endpoint") + // Call update serially. + if err := lockedCall(lockKey, updateF); err != nil { + return err + } + + log.Printf("[INFO] GKE cluster %s's enable private endpoint has been updated to %v", d.Id(), enabled) + } + if d.HasChange("binary_authorization") { req := &container.UpdateClusterRequest{ Update: &container.ClusterUpdate{ diff --git a/mmv1/third_party/terraform/tests/resource_container_cluster_test.go.erb b/mmv1/third_party/terraform/tests/resource_container_cluster_test.go.erb index 8f081122de9c..5e673c237dc8 100644 --- a/mmv1/third_party/terraform/tests/resource_container_cluster_test.go.erb +++ b/mmv1/third_party/terraform/tests/resource_container_cluster_test.go.erb @@ -4164,6 +4164,75 @@ resource "google_container_cluster" "with_private_endpoint_subnetwork" { `, containerNetName, s1Name, s1Cidr, s2Name, s2Cidr, clusterName) } +func TestAccContainerCluster_withEnablePrivateEndpointToggle(t *testing.T) { + t.Parallel() + + clusterName := fmt.Sprintf("tf-test-cluster-%s", randString(t, 10)) + + vcrTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckContainerClusterDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccContainerCluster_withEnablePrivateEndpoint(clusterName, "", ""), + }, + { + ResourceName: "google_container_cluster.with_enable_private_endpoint", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"min_master_version"}, + }, + { + Config: testAccContainerCluster_withEnablePrivateEndpoint(clusterName, "true", ""), + }, + { + ResourceName: "google_container_cluster.with_enable_private_endpoint", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"min_master_version"}, + }, + { + Config: testAccContainerCluster_withEnablePrivateEndpoint(clusterName, "false", ""), + }, + { + ResourceName: "google_container_cluster.with_enable_private_endpoint", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"min_master_version"}, + }, + }, + }) +} + +func testAccContainerCluster_withEnablePrivateEndpoint(clusterName string, flag string, emptyValue string) string { + + v := emptyValue + if flag != emptyValue { + var buf bytes.Buffer + buf.WriteString(fmt.Sprintf(` + private_cluster_config { + enable_private_endpoint = %s + }`, flag)) + v = buf.String() + } + + return fmt.Sprintf(` +resource "google_container_cluster" "with_enable_private_endpoint" { + name = "%s" + location = "us-central1-a" + min_master_version = "1.23" + initial_node_count = 1 + + master_authorized_networks_config { + gcp_public_cidrs_access_enabled = false + } + + %s +} +`, clusterName, v) +} + func testAccContainerCluster_regionalWithNodePool(cluster, nodePool string) string { return fmt.Sprintf(` resource "google_container_cluster" "regional" { From 040d79925bd6fee64351f8b217572077576cc670 Mon Sep 17 00:00:00 2001 From: Francis Liu Date: Sat, 12 Nov 2022 07:30:48 +0000 Subject: [PATCH 04/11] Add support for container cluster resource for container_node_pool.network_config.enable_private_nodes Signed-off-by: Francis Liu --- .../resource_container_node_pool.go.erb | 64 +++++++++--- .../resource_container_node_pool_test.go.erb | 98 ++++++++++++++++++- .../docs/r/container_cluster.html.markdown | 11 --- .../docs/r/container_node_pool.html.markdown | 15 ++- 4 files changed, 157 insertions(+), 31 deletions(-) diff --git a/mmv1/third_party/terraform/resources/resource_container_node_pool.go.erb b/mmv1/third_party/terraform/resources/resource_container_node_pool.go.erb index ed58a77a3072..ba523c6279a0 100644 --- a/mmv1/third_party/terraform/resources/resource_container_node_pool.go.erb +++ b/mmv1/third_party/terraform/resources/resource_container_node_pool.go.erb @@ -265,7 +265,6 @@ var schemaNodePool = map[string]*schema.Schema{ Description: `The Kubernetes version for the nodes in this pool. Note that if this field and auto_upgrade are both specified, they will fight each other for what the node version should be, so setting both is highly discouraged. While a fuzzy version can be specified, it's recommended that you specify explicit versions as Terraform will see spurious diffs when fuzzy versions are used. See the google_container_engine_versions data source's version_prefix field to approximate fuzzy versions in a Terraform-compatible way.`, }, -<% unless version == 'ga' -%> "network_config": { Type: schema.TypeList, Optional: true, @@ -280,10 +279,15 @@ var schemaNodePool = map[string]*schema.Schema{ ForceNew: true, Description: `Whether to create a new range for pod IPs in this node pool. Defaults are provided for pod_range and pod_ipv4_cidr_block if they are not specified.`, }, - + "enable_private_nodes": { + Type: schema.TypeBool, + Optional: true, + Computed: true, + Description: `Whether nodes have internal IP addresses only.`, + }, "pod_range": { Type: schema.TypeString, - Required: true, + Optional: true, ForceNew: true, Description: `The ID of the secondary range for pod IPs. If create_pod_range is true, this ID is used for the new range. If create_pod_range is false, uses an existing secondary range with this ID.`, }, @@ -298,7 +302,6 @@ var schemaNodePool = map[string]*schema.Schema{ }, }, }, -<% end -%> } @@ -772,9 +775,7 @@ func expandNodePool(d *schema.ResourceData, prefix string) (*container.NodePool, Config: expandNodeConfig(d.Get(prefix + "node_config")), Locations: locations, Version: d.Get(prefix + "version").(string), -<% unless version == 'ga' -%> NetworkConfig: expandNodeNetworkConfig(d.Get(prefix + "network_config")), -<% end -%> } if v, ok := d.GetOk(prefix + "autoscaling"); ok { @@ -878,9 +879,7 @@ func flattenNodePool(d *schema.ResourceData, config *Config, np *container.NodeP "instance_group_urls": igmUrls, "managed_instance_group_urls": managedIgmUrls, "version": np.Version, -<% unless version == 'ga' -%> "network_config": flattenNodeNetworkConfig(np.NetworkConfig, d, prefix), -<% end -%> } if np.Autoscaling != nil { @@ -934,14 +933,14 @@ func flattenNodePool(d *schema.ResourceData, config *Config, np *container.NodeP return nodePool, nil } -<% unless version == 'ga' -%> func flattenNodeNetworkConfig(c *container.NodeNetworkConfig, d *schema.ResourceData, prefix string) []map[string]interface{} { result := []map[string]interface{}{} if c != nil { result = append(result, map[string]interface{}{ - "create_pod_range": d.Get(prefix + "network_config.0.create_pod_range"), // API doesn't return this value so we set the old one. Field is ForceNew + Required - "pod_ipv4_cidr_block": c.PodIpv4CidrBlock, - "pod_range": c.PodRange, + "create_pod_range": d.Get(prefix + "network_config.0.create_pod_range"), // API doesn't return this value so we set the old one. Field is ForceNew + Required + "pod_ipv4_cidr_block": c.PodIpv4CidrBlock, + "pod_range": c.PodRange, + "enable_private_nodes": c.EnablePrivateNodes, }) } return result @@ -970,12 +969,15 @@ func expandNodeNetworkConfig(v interface{}) *container.NodeNetworkConfig { nnc.PodIpv4CidrBlock = v.(string) } + if v, ok := networkNodeConfig["enable_private_nodes"]; ok { + nnc.EnablePrivateNodes = v.(bool) + nnc.ForceSendFields = []string{"EnablePrivateNodes"} + } + return nnc } -<% end -%> - func nodePoolUpdate(d *schema.ResourceData, meta interface{}, nodePoolInfo *NodePoolInformation, prefix string, timeout time.Duration) error { config := meta.(*Config) name := d.Get(prefix + "name").(string) @@ -1393,6 +1395,40 @@ func nodePoolUpdate(d *schema.ResourceData, meta interface{}, nodePoolInfo *Node log.Printf("[INFO] Updated upgrade settings in Node Pool %s", name) } + if d.HasChange(prefix + "network_config") { + if d.HasChange(prefix + "network_config.0.enable_private_nodes") { + req := &container.UpdateNodePoolRequest{ + NodePoolId: name, + NodeNetworkConfig: expandNodeNetworkConfig(d.Get(prefix + "network_config")), + } + updateF := func() error { + clusterNodePoolsUpdateCall := config.NewContainerClient(userAgent).Projects.Locations.Clusters.NodePools.Update(nodePoolInfo.fullyQualifiedName(name),req) + if config.UserProjectOverride { + clusterNodePoolsUpdateCall.Header().Add("X-Goog-User-Project", nodePoolInfo.project) + } + op, err := clusterNodePoolsUpdateCall.Do() + + if err != nil { + return err + } + + // Wait until it's updated + return containerOperationWait(config, op, + nodePoolInfo.project, + nodePoolInfo.location, + "updating GKE node pool workload_metadata_config", userAgent, + timeout) + } + + // Call update serially. + if err := lockedCall(lockKey, updateF); err != nil { + return err + } + + log.Printf("[INFO] Updated workload_metadata_config for node pool %s", name) + } + } + return nil } diff --git a/mmv1/third_party/terraform/tests/resource_container_node_pool_test.go.erb b/mmv1/third_party/terraform/tests/resource_container_node_pool_test.go.erb index d5b35198e4fc..092df81a9b2e 100644 --- a/mmv1/third_party/terraform/tests/resource_container_node_pool_test.go.erb +++ b/mmv1/third_party/terraform/tests/resource_container_node_pool_test.go.erb @@ -447,7 +447,6 @@ func TestAccContainerNodePool_withLinuxNodeConfig(t *testing.T) { } <% end -%> -<% unless version == 'ga' -%> func TestAccContainerNodePool_withNetworkConfig(t *testing.T) { t.Parallel() @@ -478,7 +477,100 @@ func TestAccContainerNodePool_withNetworkConfig(t *testing.T) { }, }) } -<% end -%> + + +func TestAccContainerNodePool_withEnablePrivateNodesToggle(t *testing.T) { + t.Parallel() + + cluster := fmt.Sprintf("tf-test-cluster-%s", randString(t, 10)) + np := fmt.Sprintf("tf-test-np-%s", randString(t, 10)) + network := fmt.Sprintf("tf-test-net-%s", randString(t, 10)) + + vcrTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckContainerClusterDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccContainerNodePool_withEnablePrivateNodesToggle(cluster, np, network, "true"), + }, + { + ResourceName: "google_container_node_pool.with_enable_private_nodes", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"min_master_version"}, + }, + { + Config: testAccContainerNodePool_withEnablePrivateNodesToggle(cluster, np, network, "false"), + }, + { + ResourceName: "google_container_node_pool.with_enable_private_nodes", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"min_master_version"}, + }, + }, + }) +} + + +func testAccContainerNodePool_withEnablePrivateNodesToggle(cluster, np, network, flag string) string { + return fmt.Sprintf(` +resource "google_compute_network" "container_network" { + name = "%s" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "container_subnetwork" { + name = google_compute_network.container_network.name + network = google_compute_network.container_network.name + ip_cidr_range = "10.0.36.0/24" + region = "us-central1" + private_ip_google_access = true + + secondary_ip_range { + range_name = "pod" + ip_cidr_range = "10.0.0.0/19" + } + + secondary_ip_range { + range_name = "svc" + ip_cidr_range = "10.0.32.0/22" + } +} + +resource "google_container_cluster" "cluster" { + name = "%s" + location = "us-central1-a" + min_master_version = "1.23" + initial_node_count = 1 + + network = google_compute_network.container_network.name + subnetwork = google_compute_subnetwork.container_subnetwork.name + ip_allocation_policy { + cluster_secondary_range_name = google_compute_subnetwork.container_subnetwork.secondary_ip_range[0].range_name + services_secondary_range_name = google_compute_subnetwork.container_subnetwork.secondary_ip_range[1].range_name + } +} + +resource "google_container_node_pool" "with_enable_private_nodes" { + name = "%s" + location = "us-central1-a" + cluster = google_container_cluster.cluster.name + node_count = 1 + network_config { + create_pod_range = false + enable_private_nodes = %s + pod_range = google_compute_subnetwork.container_subnetwork.secondary_ip_range[0].range_name + } + node_config { + oauth_scopes = [ + "https://www.googleapis.com/auth/cloud-platform", + ] + } +} +`, network, cluster, np, flag) +} <% unless version.nil? || version == 'ga' -%> func TestAccContainerNodePool_withBootDiskKmsKey(t *testing.T) { @@ -2065,7 +2157,6 @@ resource "google_container_node_pool" "with_linux_node_config" { <% end -%> -<% unless version == 'ga' -%> func testAccContainerNodePool_withNetworkConfig(cluster, np, network string) string { return fmt.Sprintf(` resource "google_compute_network" "container_network" { @@ -2154,7 +2245,6 @@ resource "google_container_node_pool" "with_auto_pod_cidr" { `, network, cluster, np, np) } -<% end -%> <% unless version.nil? || version == 'ga' -%> diff --git a/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown b/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown index 3546c12814bf..dd8d626b5b41 100644 --- a/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown +++ b/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown @@ -255,9 +255,6 @@ region are guaranteed to support the same version. manages the default node pool, which isn't recommended to be used with Terraform. Structure is [documented below](#nested_node_config). -* `network_config` - (Optional, [Beta](https://terraform.io/docs/providers/google/guides/provider_versions.html)) Configuration for - [Adding Pod IP address ranges](https://cloud.google.com/kubernetes-engine/docs/how-to/multi-pod-cidr)) to the node pool. Structure is [documented below](#nested_network_config) - * `node_pool` - (Optional) List of node pools associated with this cluster. See [google_container_node_pool](container_node_pool.html) for schema. **Warning:** node pools defined inside a cluster can't be changed (or added/removed) after @@ -854,14 +851,6 @@ linux_node_config { * `node_group` - (Optional) Setting this field will assign instances of this pool to run on the specified node group. This is useful for running workloads on [sole tenant nodes](https://cloud.google.com/compute/docs/nodes/sole-tenant-nodes). -The `network_config` block supports: - -* `create_pod_range` - (Optional, [Beta](https://terraform.io/docs/providers/google/guides/provider_versions.html)) Whether to create a new range for pod IPs in this node pool. Defaults are provided for `pod_range` and `pod_ipv4_cidr_block` if they are not specified. - -* `pod_ipv4_cidr_block` - (Optional, [Beta](https://terraform.io/docs/providers/google/guides/provider_versions.html)) The IP address range for pod IPs in this node pool. Only applicable if createPodRange is true. Set to blank to have a range chosen with the default size. Set to /netmask (e.g. /14) to have a range chosen with a specific netmask. Set to a CIDR notation (e.g. 10.96.0.0/14) to pick a specific range to use. - -* `pod_range` - (Optional, [Beta](https://terraform.io/docs/providers/google/guides/provider_versions.html)) The ID of the secondary range for pod IPs. If `create_pod_range` is true, this ID is used for the new range. If `create_pod_range` is false, uses an existing secondary range with this ID. - The `ephemeral_storage_config` block supports: * `local_ssd_count` (Required) - Number of local SSDs to use to back ephemeral storage. Uses NVMe interfaces. Each local SSD is 375 GB in size. If zero, it means to disable using local SSDs as ephemeral storage. diff --git a/mmv1/third_party/terraform/website/docs/r/container_node_pool.html.markdown b/mmv1/third_party/terraform/website/docs/r/container_node_pool.html.markdown index e7b3b5d26ab0..d2e1d4831dd9 100644 --- a/mmv1/third_party/terraform/website/docs/r/container_node_pool.html.markdown +++ b/mmv1/third_party/terraform/website/docs/r/container_node_pool.html.markdown @@ -148,8 +148,9 @@ cluster. * `node_config` - (Optional) Parameters used in creating the node pool. See [google_container_cluster](container_cluster.html#nested_node_config) for schema. -* `network_config` - (Optional) The network configuration of the pool. See - [google_container_cluster](container_cluster.html) for schema. +* `network_config` - (Optional) The network configuration of the pool. Such as + configuration for [Adding Pod IP address ranges](https://cloud.google.com/kubernetes-engine/docs/how-to/multi-pod-cidr)) to the node pool. Or enabling private nodes. Structure is + [documented below](#nested_network_config) * `node_count` - (Optional) The number of nodes per instance group. This field can be used to update the number of nodes per instance group but should not be used alongside `autoscaling`. @@ -199,6 +200,16 @@ cluster. * `auto_upgrade` - (Optional) Whether the nodes will be automatically upgraded. +The `network_config` block supports: + +* `create_pod_range` - (Optional) Whether to create a new range for pod IPs in this node pool. Defaults are provided for `pod_range` and `pod_ipv4_cidr_block` if they are not specified. + +* `enable_private_nodes` - (Optional) Whether nodes have internal IP addresses only. + +* `pod_ipv4_cidr_block` - (Optional) The IP address range for pod IPs in this node pool. Only applicable if createPodRange is true. Set to blank to have a range chosen with the default size. Set to /netmask (e.g. /14) to have a range chosen with a specific netmask. Set to a CIDR notation (e.g. 10.96.0.0/14) to pick a specific range to use. + +* `pod_range` - (Optional) The ID of the secondary range for pod IPs. If `create_pod_range` is true, this ID is used for the new range. If `create_pod_range` is false, uses an existing secondary range with this ID. + The `upgrade_settings` block supports: * `max_surge` - (Required) The number of additional nodes that can be added to the node pool during From f367f03deb2f76ddf15e2f7eb7c347fda6d11da1 Mon Sep 17 00:00:00 2001 From: Francis Liu Date: Fri, 18 Nov 2022 01:33:35 +0000 Subject: [PATCH 05/11] Address review comments for enable_private_endpoint Signed-off-by: Francis Liu --- .../resource_container_cluster.go.erb | 50 ++++++++++++++----- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/mmv1/third_party/terraform/resources/resource_container_cluster.go.erb b/mmv1/third_party/terraform/resources/resource_container_cluster.go.erb index 321b70dc64cf..4713bc0050ce 100644 --- a/mmv1/third_party/terraform/resources/resource_container_cluster.go.erb +++ b/mmv1/third_party/terraform/resources/resource_container_cluster.go.erb @@ -82,6 +82,17 @@ var ( <% end -%> } + privateClusterConfigKeys = []string{ + "private_cluster_config.0.enable_private_endpoint", + "private_cluster_config.0.enable_private_nodes", + "private_cluster_config.0.master_ipv4_cidr_block", + "private_cluster_config.0.peering_name", + "private_cluster_config.0.private_endpoint", + "private_cluster_config.0.private_endpoint_subnetwork", + "private_cluster_config.0.public_endpoint", + "private_cluster_config.0.master_global_access_config", + } + forceNewClusterNodeConfigFields = []string{ "workload_metadata_config", } @@ -1380,9 +1391,15 @@ func resourceContainerCluster() *schema.Resource { Description: `Configuration for private clusters, clusters with private nodes.`, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ + // enable_private_endpoint is orthogonal to private_endpoint_subnetwork. + // User can create a private_cluster_config block without including + // either one of those two fields. Both fields are optional. + // At the same time, we use 'AtLeastOneOf' to prevent an empty block + // like 'private_cluster_config{}' "enable_private_endpoint": { Type: schema.TypeBool, Optional: true, + AtLeastOneOf: privateClusterConfigKeys, DiffSuppressFunc: containerClusterPrivateClusterConfigSuppress, Description: `When true, the cluster's private endpoint is used as the cluster endpoint and access through the public endpoint is disabled. When false, either endpoint can be used. This field only applies to private clusters, when enable_private_nodes is true.`, }, @@ -1390,6 +1407,7 @@ func resourceContainerCluster() *schema.Resource { Type: schema.TypeBool, Optional: true, ForceNew: true, + AtLeastOneOf: privateClusterConfigKeys, DiffSuppressFunc: containerClusterPrivateClusterConfigSuppress, Description: `Enables the private cluster feature, creating a private endpoint on the cluster. In a private cluster, nodes only have RFC 1918 private addresses and communicate with the master's private endpoint via private networking.`, }, @@ -1398,36 +1416,42 @@ func resourceContainerCluster() *schema.Resource { Computed: true, Optional: true, ForceNew: true, + AtLeastOneOf: privateClusterConfigKeys, ValidateFunc: orEmpty(validation.IsCIDRNetwork(28, 28)), Description: `The IP range in CIDR notation to use for the hosted master network. This range will be used for assigning private IP addresses to the cluster master(s) and the ILB VIP. This range must not overlap with any other ranges in use within the cluster's network, and it must be a /28 subnet. See Private Cluster Limitations for more details. This field only applies to private clusters, when enable_private_nodes is true.`, }, "peering_name": { - Type: schema.TypeString, - Computed: true, + Type: schema.TypeString, + Computed: true, + AtLeastOneOf: privateClusterConfigKeys, Description: `The name of the peering between this cluster and the Google owned VPC.`, }, "private_endpoint": { - Type: schema.TypeString, - Computed: true, + Type: schema.TypeString, + Computed: true, + AtLeastOneOf: privateClusterConfigKeys, Description: `The internal IP address of this cluster's master endpoint.`, }, "private_endpoint_subnetwork": { - Type: schema.TypeString, - Optional: true, - ForceNew: true, + Type: schema.TypeString, + Optional: true, + ForceNew: true, + AtLeastOneOf: privateClusterConfigKeys, DiffSuppressFunc: compareSelfLinkOrResourceName, Description: `Subnetwork in cluster's network where master's endpoint will be provisioned.`, }, "public_endpoint": { - Type: schema.TypeString, - Computed: true, + Type: schema.TypeString, + Computed: true, + AtLeastOneOf: privateClusterConfigKeys, Description: `The external IP address of this cluster's master endpoint.`, }, "master_global_access_config": { - Type: schema.TypeList, - MaxItems: 1, - Optional: true, - Computed: true, + Type: schema.TypeList, + MaxItems: 1, + Optional: true, + Computed: true, + AtLeastOneOf: privateClusterConfigKeys, Description: "Controls cluster master global access settings.", Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ From d1d5d37b81c1f59c12209c6874cd9eabfbb0eaeb Mon Sep 17 00:00:00 2001 From: Francis Liu Date: Fri, 18 Nov 2022 01:41:41 +0000 Subject: [PATCH 06/11] Address the enable_private_endpoint comment Signed-off-by: Francis Liu --- .../resource_container_cluster_test.go.erb | 40 +++++++++++-------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/mmv1/third_party/terraform/tests/resource_container_cluster_test.go.erb b/mmv1/third_party/terraform/tests/resource_container_cluster_test.go.erb index 5e673c237dc8..cc9eecb0a8e9 100644 --- a/mmv1/third_party/terraform/tests/resource_container_cluster_test.go.erb +++ b/mmv1/third_party/terraform/tests/resource_container_cluster_test.go.erb @@ -4175,7 +4175,7 @@ func TestAccContainerCluster_withEnablePrivateEndpointToggle(t *testing.T) { CheckDestroy: testAccCheckContainerClusterDestroyProducer(t), Steps: []resource.TestStep{ { - Config: testAccContainerCluster_withEnablePrivateEndpoint(clusterName, "", ""), + Config: testAccContainerCluster_withoutEnablePrivateEndpoint(clusterName), }, { ResourceName: "google_container_cluster.with_enable_private_endpoint", @@ -4184,7 +4184,7 @@ func TestAccContainerCluster_withEnablePrivateEndpointToggle(t *testing.T) { ImportStateVerifyIgnore: []string{"min_master_version"}, }, { - Config: testAccContainerCluster_withEnablePrivateEndpoint(clusterName, "true", ""), + Config: testAccContainerCluster_withEnablePrivateEndpoint(clusterName, "true"), }, { ResourceName: "google_container_cluster.with_enable_private_endpoint", @@ -4193,7 +4193,7 @@ func TestAccContainerCluster_withEnablePrivateEndpointToggle(t *testing.T) { ImportStateVerifyIgnore: []string{"min_master_version"}, }, { - Config: testAccContainerCluster_withEnablePrivateEndpoint(clusterName, "false", ""), + Config: testAccContainerCluster_withEnablePrivateEndpoint(clusterName, "false"), }, { ResourceName: "google_container_cluster.with_enable_private_endpoint", @@ -4205,17 +4205,7 @@ func TestAccContainerCluster_withEnablePrivateEndpointToggle(t *testing.T) { }) } -func testAccContainerCluster_withEnablePrivateEndpoint(clusterName string, flag string, emptyValue string) string { - - v := emptyValue - if flag != emptyValue { - var buf bytes.Buffer - buf.WriteString(fmt.Sprintf(` - private_cluster_config { - enable_private_endpoint = %s - }`, flag)) - v = buf.String() - } +func testAccContainerCluster_withEnablePrivateEndpoint(clusterName string, flag string) string { return fmt.Sprintf(` resource "google_container_cluster" "with_enable_private_endpoint" { @@ -4228,9 +4218,27 @@ resource "google_container_cluster" "with_enable_private_endpoint" { gcp_public_cidrs_access_enabled = false } - %s + private_cluster_config { + enable_private_endpoint = %s + } } -`, clusterName, v) +`, clusterName, flag) +} + +func testAccContainerCluster_withoutEnablePrivateEndpoint(clusterName string) string { + + return fmt.Sprintf(` +resource "google_container_cluster" "with_enable_private_endpoint" { + name = "%s" + location = "us-central1-a" + min_master_version = "1.23" + initial_node_count = 1 + + master_authorized_networks_config { + gcp_public_cidrs_access_enabled = false + } +} +`, clusterName) } func testAccContainerCluster_regionalWithNodePool(cluster, nodePool string) string { From 0d702eb6c6158910a99f2a58abe7eec4eaabd8c8 Mon Sep 17 00:00:00 2001 From: Francis Liu Date: Fri, 18 Nov 2022 01:47:29 +0000 Subject: [PATCH 07/11] Address review comments for GcpPublicCidrsAccessEnabled Signed-off-by: Francis Liu --- .../resource_container_cluster_test.go.erb | 58 ++++++++++--------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/mmv1/third_party/terraform/tests/resource_container_cluster_test.go.erb b/mmv1/third_party/terraform/tests/resource_container_cluster_test.go.erb index cc9eecb0a8e9..f61068f05b49 100644 --- a/mmv1/third_party/terraform/tests/resource_container_cluster_test.go.erb +++ b/mmv1/third_party/terraform/tests/resource_container_cluster_test.go.erb @@ -725,7 +725,7 @@ func TestAccContainerCluster_withGcpPublicCidrsAccessEnabledToggle(t *testing.T) CheckDestroy: testAccCheckContainerClusterDestroyProducer(t), Steps: []resource.TestStep{ { - Config: testAccContainerCluster_withGcpPublicCidrsAccessEnabled(clusterName, "", ""), + Config: testAccContainerCluster_withoutGcpPublicCidrsAccessEnabled(clusterName), Check: resource.ComposeTestCheckFunc( resource.TestCheckResourceAttr("google_container_cluster.with_gcp_public_cidrs_access_enabled", "master_authorized_networks_config.#", "1"), @@ -740,7 +740,7 @@ func TestAccContainerCluster_withGcpPublicCidrsAccessEnabledToggle(t *testing.T) ImportStateVerifyIgnore: []string{"min_master_version"}, }, { - Config: testAccContainerCluster_withGcpPublicCidrsAccessEnabled(clusterName, "false", ""), + Config: testAccContainerCluster_withGcpPublicCidrsAccessEnabled(clusterName, "false"), Check: resource.ComposeTestCheckFunc( resource.TestCheckResourceAttr("google_container_cluster.with_gcp_public_cidrs_access_enabled", "master_authorized_networks_config.0.gcp_public_cidrs_access_enabled", "false"), @@ -753,7 +753,7 @@ func TestAccContainerCluster_withGcpPublicCidrsAccessEnabledToggle(t *testing.T) ImportStateVerifyIgnore: []string{"min_master_version"}, }, { - Config: testAccContainerCluster_withGcpPublicCidrsAccessEnabled(clusterName, "true", ""), + Config: testAccContainerCluster_withGcpPublicCidrsAccessEnabled(clusterName, "true"), Check: resource.ComposeTestCheckFunc( resource.TestCheckResourceAttr("google_container_cluster.with_gcp_public_cidrs_access_enabled", "master_authorized_networks_config.0.gcp_public_cidrs_access_enabled", "true"), @@ -763,6 +763,34 @@ func TestAccContainerCluster_withGcpPublicCidrsAccessEnabledToggle(t *testing.T) }) } +func testAccContainerCluster_withGcpPublicCidrsAccessEnabled(clusterName string, flag string) string { + + return fmt.Sprintf(` +resource "google_container_cluster" "with_gcp_public_cidrs_access_enabled" { + name = "%s" + location = "us-central1-a" + min_master_version = "1.23" + initial_node_count = 1 + + master_authorized_networks_config { + gcp_public_cidrs_access_enabled = %s + } +} +`, clusterName, flag) +} + +func testAccContainerCluster_withoutGcpPublicCidrsAccessEnabled(clusterName string) string { + + return fmt.Sprintf(` +resource "google_container_cluster" "with_gcp_public_cidrs_access_enabled" { + name = "%s" + location = "us-central1-a" + min_master_version = "1.23" + initial_node_count = 1 +} +`, clusterName) +} + func TestAccContainerCluster_regional(t *testing.T) { t.Parallel() @@ -4049,30 +4077,6 @@ resource "google_container_cluster" "with_master_authorized_networks" { `, clusterName, cidrBlocks) } -func testAccContainerCluster_withGcpPublicCidrsAccessEnabled(clusterName string, flag string, emptyValue string) string { - - v := emptyValue - if flag != emptyValue { - var buf bytes.Buffer - buf.WriteString(fmt.Sprintf(` - master_authorized_networks_config { - gcp_public_cidrs_access_enabled = %s - }`, flag)) - v = buf.String() - } - - return fmt.Sprintf(` -resource "google_container_cluster" "with_gcp_public_cidrs_access_enabled" { - name = "%s" - location = "us-central1-a" - min_master_version = "1.23" - initial_node_count = 1 - - %s -} -`, clusterName, v) -} - func testAccContainerCluster_removeMasterAuthorizedNetworksConfig(clusterName string) string { return fmt.Sprintf(` resource "google_container_cluster" "with_master_authorized_networks" { From 4d77b53d8c7e2b932a2a61dc4e4e1952d277fbb1 Mon Sep 17 00:00:00 2001 From: Francis Liu Date: Fri, 18 Nov 2022 16:21:10 +0000 Subject: [PATCH 08/11] Fix AtLeastOneOf: excluding output-only fields Signed-off-by: Francis Liu --- .../terraform/resources/resource_container_cluster.go.erb | 6 ------ 1 file changed, 6 deletions(-) diff --git a/mmv1/third_party/terraform/resources/resource_container_cluster.go.erb b/mmv1/third_party/terraform/resources/resource_container_cluster.go.erb index 4713bc0050ce..b8d7f1d8051f 100644 --- a/mmv1/third_party/terraform/resources/resource_container_cluster.go.erb +++ b/mmv1/third_party/terraform/resources/resource_container_cluster.go.erb @@ -86,10 +86,7 @@ var ( "private_cluster_config.0.enable_private_endpoint", "private_cluster_config.0.enable_private_nodes", "private_cluster_config.0.master_ipv4_cidr_block", - "private_cluster_config.0.peering_name", - "private_cluster_config.0.private_endpoint", "private_cluster_config.0.private_endpoint_subnetwork", - "private_cluster_config.0.public_endpoint", "private_cluster_config.0.master_global_access_config", } @@ -1423,13 +1420,11 @@ func resourceContainerCluster() *schema.Resource { "peering_name": { Type: schema.TypeString, Computed: true, - AtLeastOneOf: privateClusterConfigKeys, Description: `The name of the peering between this cluster and the Google owned VPC.`, }, "private_endpoint": { Type: schema.TypeString, Computed: true, - AtLeastOneOf: privateClusterConfigKeys, Description: `The internal IP address of this cluster's master endpoint.`, }, "private_endpoint_subnetwork": { @@ -1443,7 +1438,6 @@ func resourceContainerCluster() *schema.Resource { "public_endpoint": { Type: schema.TypeString, Computed: true, - AtLeastOneOf: privateClusterConfigKeys, Description: `The external IP address of this cluster's master endpoint.`, }, "master_global_access_config": { From 45555e27b0e8a533c8f7bb82bd5ba3c554df1046 Mon Sep 17 00:00:00 2001 From: Francis Liu Date: Tue, 22 Nov 2022 17:47:21 +0000 Subject: [PATCH 09/11] Address more comments for GcpPublicCidrsAccessEnabled Signed-off-by: Francis Liu --- .../resource_container_cluster.go.erb | 18 ++++++++---------- .../docs/r/container_cluster.html.markdown | 2 +- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/mmv1/third_party/terraform/resources/resource_container_cluster.go.erb b/mmv1/third_party/terraform/resources/resource_container_cluster.go.erb index b8d7f1d8051f..52037ea737c8 100644 --- a/mmv1/third_party/terraform/resources/resource_container_cluster.go.erb +++ b/mmv1/third_party/terraform/resources/resource_container_cluster.go.erb @@ -5004,20 +5004,18 @@ func flattenStandardRolloutPolicy(a *container.StandardRolloutPolicy) []map[stri } func flattenMasterAuthorizedNetworksConfig(c *container.MasterAuthorizedNetworksConfig) []map[string]interface{} { - if c == nil { + if c == nil || !c.Enabled { return nil } result := make(map[string]interface{}) - if c.Enabled { - cidrBlocks := make([]interface{}, 0, len(c.CidrBlocks)) - for _, v := range c.CidrBlocks { - cidrBlocks = append(cidrBlocks, map[string]interface{}{ - "cidr_block": v.CidrBlock, - "display_name": v.DisplayName, - }) - } - result["cidr_blocks"] = schema.NewSet(schema.HashResource(cidrBlockConfig), cidrBlocks) + cidrBlocks := make([]interface{}, 0, len(c.CidrBlocks)) + for _, v := range c.CidrBlocks { + cidrBlocks = append(cidrBlocks, map[string]interface{}{ + "cidr_block": v.CidrBlock, + "display_name": v.DisplayName, + }) } + result["cidr_blocks"] = schema.NewSet(schema.HashResource(cidrBlockConfig), cidrBlocks) result["gcp_public_cidrs_access_enabled"] = c.GcpPublicCidrsAccessEnabled return []map[string]interface{}{result} } diff --git a/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown b/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown index dd8d626b5b41..c8663f4954c7 100644 --- a/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown +++ b/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown @@ -693,7 +693,7 @@ This block also contains several computed attributes, documented below. * `cidr_blocks` - (Optional) External networks that can access the Kubernetes cluster master through HTTPS. -* `with_gcp_public_cidrs_access_enabled` - (Optional) Whether Kubernetes master is +* `gcp_public_cidrs_access_enabled` - (Optional) Whether Kubernetes master is accessible via Google Compute Engine Public IPs. The `master_authorized_networks_config.cidr_blocks` block supports: From 4854f9ab5519d1a26bcb1fb74aa83186a60f577f Mon Sep 17 00:00:00 2001 From: Francis Liu Date: Tue, 22 Nov 2022 18:08:11 +0000 Subject: [PATCH 10/11] Fix GcpPublicCidrsAccessEnabled test case Signed-off-by: Francis Liu --- .../terraform/tests/resource_container_cluster_test.go.erb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mmv1/third_party/terraform/tests/resource_container_cluster_test.go.erb b/mmv1/third_party/terraform/tests/resource_container_cluster_test.go.erb index f61068f05b49..ce7f91983aac 100644 --- a/mmv1/third_party/terraform/tests/resource_container_cluster_test.go.erb +++ b/mmv1/third_party/terraform/tests/resource_container_cluster_test.go.erb @@ -728,9 +728,7 @@ func TestAccContainerCluster_withGcpPublicCidrsAccessEnabledToggle(t *testing.T) Config: testAccContainerCluster_withoutGcpPublicCidrsAccessEnabled(clusterName), Check: resource.ComposeTestCheckFunc( resource.TestCheckResourceAttr("google_container_cluster.with_gcp_public_cidrs_access_enabled", - "master_authorized_networks_config.#", "1"), - resource.TestCheckResourceAttr("google_container_cluster.with_gcp_public_cidrs_access_enabled", - "master_authorized_networks_config.0.gcp_public_cidrs_access_enabled", "true"), + "master_authorized_networks_config.#", "0"), ), }, { From eff883104d2126723b408a91df7b6a5c23361a8f Mon Sep 17 00:00:00 2001 From: Francis Liu Date: Mon, 28 Nov 2022 18:29:06 +0000 Subject: [PATCH 11/11] Fixed spacing Signed-off-by: Francis Liu --- .../terraform/tests/resource_container_cluster_test.go.erb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mmv1/third_party/terraform/tests/resource_container_cluster_test.go.erb b/mmv1/third_party/terraform/tests/resource_container_cluster_test.go.erb index ce7f91983aac..d88b20bdcb38 100644 --- a/mmv1/third_party/terraform/tests/resource_container_cluster_test.go.erb +++ b/mmv1/third_party/terraform/tests/resource_container_cluster_test.go.erb @@ -771,7 +771,7 @@ resource "google_container_cluster" "with_gcp_public_cidrs_access_enabled" { initial_node_count = 1 master_authorized_networks_config { - gcp_public_cidrs_access_enabled = %s + gcp_public_cidrs_access_enabled = %s } } `, clusterName, flag) @@ -4160,7 +4160,7 @@ resource "google_container_cluster" "with_private_endpoint_subnetwork" { subnetwork = google_compute_subnetwork.container_subnetwork1.name private_cluster_config { - private_endpoint_subnetwork = google_compute_subnetwork.container_subnetwork2.name + private_endpoint_subnetwork = google_compute_subnetwork.container_subnetwork2.name } } `, containerNetName, s1Name, s1Cidr, s2Name, s2Cidr, clusterName) @@ -4221,7 +4221,7 @@ resource "google_container_cluster" "with_enable_private_endpoint" { } private_cluster_config { - enable_private_endpoint = %s + enable_private_endpoint = %s } } `, clusterName, flag)