Skip to content

Commit

Permalink
Add support for flexible clusters - public/private clusters toggle (#…
Browse files Browse the repository at this point in the history 
…6780) (#13128)

Signed-off-by: Modular Magician <magic-modules@google.com>

Signed-off-by: Modular Magician <magic-modules@google.com>
  • Loading branch information
@modular-magician
modular-magician authored Nov 28, 2022
1 parent d02c5c7 commit 7189bc4
Show file tree
Hide file tree
Showing 7 changed files with 671 additions and 44 deletions.
12 changes: 12 additions & 0 deletions .changelog/6780.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
```release-note:enhancement
container: added field `gcp_public_cidrs_access_enabled` and `private_endpoint_subnetwork` to `google_container_cluster`
```
```release-note:enhancement
container: added update support for `enable_private_endpoint` and `enable_private_nodes` in `google_container_cluster`
```
```release-note:enhancement
container: promoted `network_config` in `google_container_node_pool` to GA.
```
```release-note:enhancement
container: added field `enable_private_nodes` in `network_config` to `google_container_node_pool`
```
119 changes: 88 additions & 31 deletions google/resource_container_cluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,17 +22,23 @@ import (
var (
instanceGroupManagerURL = regexp.MustCompile(fmt.Sprintf("projects/(%s)/zones/([a-z0-9-]*)/instanceGroupManagers/([^/]*)", ProjectRegex))

networkConfig = &schema.Resource{
masterAuthorizedNetworksConfig = &schema.Resource{
Schema: map[string]*schema.Schema{
"cidr_blocks": {
Type: schema.TypeSet,
// Despite being the only entry in a nested block, this should be kept
// Optional. Expressing the parent with no entries and omitting the
// This should be kept Optional. Expressing the
// parent with no entries and omitting the
// parent entirely are semantically different.
Optional: true,
Elem: cidrBlockConfig,
Description: `External networks that can access the Kubernetes cluster master through HTTPS.`,
},
"gcp_public_cidrs_access_enabled": {
Type: schema.TypeBool,
Optional: true,
Computed: true,
Description: `Whether master is accessbile via Google Compute Engine Public IP addresses.`,
},
},
}
cidrBlockConfig = &schema.Resource{
Expand Down Expand Up @@ -64,6 +70,14 @@ var (
"addons_config.0.gce_persistent_disk_csi_driver_config",
}

privateClusterConfigKeys = []string{
"private_cluster_config.0.enable_private_endpoint",
"private_cluster_config.0.enable_private_nodes",
"private_cluster_config.0.master_ipv4_cidr_block",
"private_cluster_config.0.private_endpoint_subnetwork",
"private_cluster_config.0.master_global_access_config",
}

forceNewClusterNodeConfigFields = []string{
"workload_metadata_config",
}
Expand Down Expand Up @@ -918,8 +932,9 @@ func resourceContainerCluster() *schema.Resource {
"master_authorized_networks_config": {
Type: schema.TypeList,
Optional: true,
Computed: true,
MaxItems: 1,
Elem: networkConfig,
Elem: masterAuthorizedNetworksConfig,
Description: `The desired configuration options for master authorized networks. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists).`,
},

Expand Down Expand Up @@ -1114,17 +1129,23 @@ func resourceContainerCluster() *schema.Resource {
Description: `Configuration for private clusters, clusters with private nodes.`,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
// enable_private_endpoint is orthogonal to private_endpoint_subnetwork.
// User can create a private_cluster_config block without including
// either one of those two fields. Both fields are optional.
// At the same time, we use 'AtLeastOneOf' to prevent an empty block
// like 'private_cluster_config{}'
"enable_private_endpoint": {
Type: schema.TypeBool,
Required: true,
ForceNew: true,
Optional: true,
AtLeastOneOf: privateClusterConfigKeys,
DiffSuppressFunc: containerClusterPrivateClusterConfigSuppress,
Description: `When true, the cluster's private endpoint is used as the cluster endpoint and access through the public endpoint is disabled. When false, either endpoint can be used. This field only applies to private clusters, when enable_private_nodes is true.`,
},
"enable_private_nodes": {
Type: schema.TypeBool,
Optional: true,
ForceNew: true,
AtLeastOneOf: privateClusterConfigKeys,
DiffSuppressFunc: containerClusterPrivateClusterConfigSuppress,
Description: `Enables the private cluster feature, creating a private endpoint on the cluster. In a private cluster, nodes only have RFC 1918 private addresses and communicate with the master's private endpoint via private networking.`,
},
Expand All @@ -1133,6 +1154,7 @@ func resourceContainerCluster() *schema.Resource {
Computed: true,
Optional: true,
ForceNew: true,
AtLeastOneOf: privateClusterConfigKeys,
ValidateFunc: orEmpty(validation.IsCIDRNetwork(28, 28)),
Description: `The IP range in CIDR notation to use for the hosted master network. This range will be used for assigning private IP addresses to the cluster master(s) and the ILB VIP. This range must not overlap with any other ranges in use within the cluster's network, and it must be a /28 subnet. See Private Cluster Limitations for more details. This field only applies to private clusters, when enable_private_nodes is true.`,
},
Expand All @@ -1146,17 +1168,26 @@ func resourceContainerCluster() *schema.Resource {
Computed: true,
Description: `The internal IP address of this cluster's master endpoint.`,
},
"private_endpoint_subnetwork": {
Type: schema.TypeString,
Optional: true,
ForceNew: true,
AtLeastOneOf: privateClusterConfigKeys,
DiffSuppressFunc: compareSelfLinkOrResourceName,
Description: `Subnetwork in cluster's network where master's endpoint will be provisioned.`,
},
"public_endpoint": {
Type: schema.TypeString,
Computed: true,
Description: `The external IP address of this cluster's master endpoint.`,
},
"master_global_access_config": {
Type: schema.TypeList,
MaxItems: 1,
Optional: true,
Computed: true,
Description: "Controls cluster master global access settings.",
Type: schema.TypeList,
MaxItems: 1,
Optional: true,
Computed: true,
AtLeastOneOf: privateClusterConfigKeys,
Description: "Controls cluster master global access settings.",
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"enabled": {
Expand Down Expand Up @@ -1535,7 +1566,7 @@ func resourceContainerClusterCreate(d *schema.ResourceData, meta interface{}) er
Name: clusterName,
InitialNodeCount: int64(d.Get("initial_node_count").(int)),
MaintenancePolicy: expandMaintenancePolicy(d, meta),
MasterAuthorizedNetworksConfig: expandMasterAuthorizedNetworksConfig(d.Get("master_authorized_networks_config")),
MasterAuthorizedNetworksConfig: expandMasterAuthorizedNetworksConfig(d.Get("master_authorized_networks_config"), d),
InitialClusterVersion: d.Get("min_master_version").(string),
ClusterIpv4Cidr: d.Get("cluster_ipv4_cidr").(string),
Description: d.Get("description").(string),
Expand Down Expand Up @@ -2098,7 +2129,7 @@ func resourceContainerClusterUpdate(d *schema.ResourceData, meta interface{}) er
c := d.Get("master_authorized_networks_config")
req := &container.UpdateClusterRequest{
Update: &container.ClusterUpdate{
DesiredMasterAuthorizedNetworksConfig: expandMasterAuthorizedNetworksConfig(c),
DesiredMasterAuthorizedNetworksConfig: expandMasterAuthorizedNetworksConfig(c, d),
},
}

Expand Down Expand Up @@ -2162,6 +2193,24 @@ func resourceContainerClusterUpdate(d *schema.ResourceData, meta interface{}) er
log.Printf("[INFO] GKE cluster %s's binary authorization has been updated to %v", d.Id(), enabled)
}

if d.HasChange("private_cluster_config.0.enable_private_endpoint") {
enabled := d.Get("private_cluster_config.0.enable_private_endpoint").(bool)
req := &container.UpdateClusterRequest{
Update: &container.ClusterUpdate{
DesiredEnablePrivateEndpoint: enabled,
ForceSendFields: []string{"DesiredEnablePrivateEndpoint"},
},
}

updateF := updateFunc(req, "updating enable private endpoint")
// Call update serially.
if err := lockedCall(lockKey, updateF); err != nil {
return err
}

log.Printf("[INFO] GKE cluster %s's enable private endpoint has been updated to %v", d.Id(), enabled)
}

if d.HasChange("binary_authorization") {
req := &container.UpdateClusterRequest{
Update: &container.ClusterUpdate{
Expand Down Expand Up @@ -3537,7 +3586,7 @@ func expandMasterAuth(configured interface{}) *container.MasterAuth {
return result
}

func expandMasterAuthorizedNetworksConfig(configured interface{}) *container.MasterAuthorizedNetworksConfig {
func expandMasterAuthorizedNetworksConfig(configured interface{}, d *schema.ResourceData) *container.MasterAuthorizedNetworksConfig {
l := configured.([]interface{})
if len(l) == 0 {
return &container.MasterAuthorizedNetworksConfig{
Expand All @@ -3559,6 +3608,10 @@ func expandMasterAuthorizedNetworksConfig(configured interface{}) *container.Mas
})
}
}
if v, ok := d.GetOkExists("master_authorized_networks_config.0.gcp_public_cidrs_access_enabled"); ok {
result.GcpPublicCidrsAccessEnabled = v.(bool)
result.ForceSendFields = []string{"GcpPublicCidrsAccessEnabled"}
}
}
return result
}
Expand Down Expand Up @@ -3586,11 +3639,12 @@ func expandPrivateClusterConfig(configured interface{}) *container.PrivateCluste
}
config := l[0].(map[string]interface{})
return &container.PrivateClusterConfig{
EnablePrivateEndpoint: config["enable_private_endpoint"].(bool),
EnablePrivateNodes: config["enable_private_nodes"].(bool),
MasterIpv4CidrBlock: config["master_ipv4_cidr_block"].(string),
MasterGlobalAccessConfig: expandPrivateClusterConfigMasterGlobalAccessConfig(config["master_global_access_config"]),
ForceSendFields: []string{"EnablePrivateEndpoint", "EnablePrivateNodes", "MasterIpv4CidrBlock", "MasterGlobalAccessConfig"},
EnablePrivateEndpoint: config["enable_private_endpoint"].(bool),
EnablePrivateNodes: config["enable_private_nodes"].(bool),
MasterIpv4CidrBlock: config["master_ipv4_cidr_block"].(string),
MasterGlobalAccessConfig: expandPrivateClusterConfigMasterGlobalAccessConfig(config["master_global_access_config"]),
PrivateEndpointSubnetwork: config["private_endpoint_subnetwork"].(string),
ForceSendFields: []string{"EnablePrivateEndpoint", "EnablePrivateNodes", "MasterIpv4CidrBlock", "MasterGlobalAccessConfig"},
}
}

Expand Down Expand Up @@ -4020,6 +4074,7 @@ func flattenPrivateClusterConfig(c *container.PrivateClusterConfig) []map[string
"master_global_access_config": flattenPrivateClusterConfigMasterGlobalAccessConfig(c.MasterGlobalAccessConfig),
"peering_name": c.PeeringName,
"private_endpoint": c.PrivateEndpoint,
"private_endpoint_subnetwork": c.PrivateEndpointSubnetwork,
"public_endpoint": c.PublicEndpoint,
},
}
Expand Down Expand Up @@ -4263,16 +4318,15 @@ func flattenMasterAuthorizedNetworksConfig(c *container.MasterAuthorizedNetworks
return nil
}
result := make(map[string]interface{})
if c.Enabled {
cidrBlocks := make([]interface{}, 0, len(c.CidrBlocks))
for _, v := range c.CidrBlocks {
cidrBlocks = append(cidrBlocks, map[string]interface{}{
"cidr_block": v.CidrBlock,
"display_name": v.DisplayName,
})
}
result["cidr_blocks"] = schema.NewSet(schema.HashResource(cidrBlockConfig), cidrBlocks)
cidrBlocks := make([]interface{}, 0, len(c.CidrBlocks))
for _, v := range c.CidrBlocks {
cidrBlocks = append(cidrBlocks, map[string]interface{}{
"cidr_block": v.CidrBlock,
"display_name": v.DisplayName,
})
}
result["cidr_blocks"] = schema.NewSet(schema.HashResource(cidrBlockConfig), cidrBlocks)
result["gcp_public_cidrs_access_enabled"] = c.GcpPublicCidrsAccessEnabled
return []map[string]interface{}{result}
}

Expand Down Expand Up @@ -4491,12 +4545,15 @@ func containerClusterPrivateClusterConfigSuppress(k, old, new string, d *schema.
o, n = d.GetChange("private_cluster_config.0.enable_private_nodes")
suppressNodes := !o.(bool) && !n.(bool)

// Do not suppress diffs when private_endpoint_subnetwork is configured
_, hasSubnet := d.GetOk("private_cluster_config.0.private_endpoint_subnetwork")

if k == "private_cluster_config.0.enable_private_endpoint" {
return suppressEndpoint
return suppressEndpoint && !hasSubnet
} else if k == "private_cluster_config.0.enable_private_nodes" {
return suppressNodes
return suppressNodes && !hasSubnet
} else if k == "private_cluster_config.#" {
return suppressEndpoint && suppressNodes
return suppressEndpoint && suppressNodes && !hasSubnet
}
return false
}
Expand Down
Loading

0 comments on commit 7189bc4

Please sign in to comment.