Skip to content

Commit

Permalink
wrap up initial kubelet/linux config support. Add some tests
Browse files Browse the repository at this point in the history
  • Loading branch information
rmanyari committed Jul 20, 2020
1 parent ada5392 commit 812a880
Show file tree
Hide file tree
Showing 3 changed files with 272 additions and 15 deletions.
66 changes: 51 additions & 15 deletions google-beta/node_config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import (
"math"
"regexp"
"strconv"
"strings"

"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
"github.com/hashicorp/terraform-plugin-sdk/helper/validation"
Expand All @@ -24,6 +23,20 @@ var defaultOauthScopes = []string{
// This validates for a tuple-3 of positive integers. Spaces and tabs are supported as separators
var tcpMemoryRegexp = regexp.MustCompile(`^([1-9][0-9]+|0)[ \t]+([1-9][0-9]+|0)[ \t]+([1-9][0-9]+|0)$`)

// Keep track of the original sysctl keys since we replace dots by underscores to make the keys more TF idiomatic.
// Ref: https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/NodeConfig#LinuxNodeConfig
var sysAttrsMap = map[string]string{
"net_core_netdev_max_backlog": "net.core.netdev_max_backlog",
"net_core_rmem_max": "net.core.rmem_max",
"net_core_wmem_default": "net.core.wmem_default",
"net_core_wmem_max": "net.core.wmem_max",
"net_core_optmem_max": "net.core.optmem_max",
"net_core_somaxconn": "net.core.somaxconn",
"net_ipv4_tcp_rmem": "net.ipv4.tcp_rmem",
"net_ipv4_tcp_wmem": "net.ipv4.tcp_wmem",
"net_ipv4_tcp_tw_reuse": "net.ipv4.tcp_tw_reuse",
}

func schemaNodeConfig() *schema.Schema {
return &schema.Schema{
Type: schema.TypeList,
Expand Down Expand Up @@ -281,51 +294,51 @@ func schemaNodeConfig() *schema.Schema {
Required: true,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"net-core-netdev_max_backlog": {
"net_core_netdev_max_backlog": {
Type: schema.TypeInt,
Optional: true,
ValidateFunc: validation.IntBetween(1, math.MaxInt32),
},
"net-core-rmem_max": {
"net_core_rmem_max": {
Type: schema.TypeInt,
Optional: true,
ValidateFunc: validation.IntBetween(1, math.MaxInt32),
},
"net-core-wmem_default": {
"net_core_wmem_default": {
Type: schema.TypeInt,
Optional: true,
ValidateFunc: validation.IntBetween(1, math.MaxInt32),
},
"net-core-wmem_max": {
"net_core_wmem_max": {
Type: schema.TypeInt,
Optional: true,
ValidateFunc: validation.IntBetween(1, math.MaxInt32),
},
"net-core-optmem_max": {
"net_core_optmem_max": {
Type: schema.TypeInt,
Optional: true,
ValidateFunc: validation.IntBetween(1, math.MaxInt32),
},
"net-core-somaxconn": {
"net_core_somaxconn": {
Type: schema.TypeInt,
Optional: true,
ValidateFunc: validation.IntBetween(128, math.MaxInt32),
},
"net-ipv4-tcp_rmem": {
"net_ipv4_tcp_rmem": {
Type: schema.TypeString,
Optional: true,
ValidateFunc: validation.StringMatch(
tcpMemoryRegexp,
"net-ipv4-tcp_rmem must be a tuple-3 of positive integers separated by spaces and or tabs"),
"net_ipv4_tcp_rmem must be a tuple-3 of positive integers separated by spaces and or tabs"),
},
"net-ipv4-tcp_wmem": {
"net_ipv4_tcp_wmem": {
Type: schema.TypeString,
Optional: true,
ValidateFunc: validation.StringMatch(
tcpMemoryRegexp,
"net-ipv4-tcp_wmem must be a tuple-3 of positive integers separated by spaces and or tabs"),
"net_ipv4_tcp_wmem must be a tuple-3 of positive integers separated by spaces and or tabs"),
},
"net-ipv4-tcp_tw_reuse": {
"net_ipv4_tcp_tw_reuse": {
Type: schema.TypeInt,
Optional: true,
ValidateFunc: validation.IntBetween(0, 1),
Expand Down Expand Up @@ -550,10 +563,18 @@ func expandLinuxNodeConfig(v interface{}) *containerBeta.LinuxNodeConfig {
if s, ok := v.(string); ok {
casted = s
}
// So far the schema has only strings and ints.
// If we get anything other than that we skip it.
// If we ever support more types than that, we
// will need to add support for it here.
if casted == "" {
continue
}
sysctls[strings.ReplaceAll(k, "-", ".")] = casted
// Lookup the actual sysctl supported key name
// to be sent to GCP.
if sysK, ok := sysAttrsMap[k]; ok {
sysctls[sysK] = casted
}
}
return &containerBeta.LinuxNodeConfig{
Sysctls: sysctls,
Expand Down Expand Up @@ -665,9 +686,24 @@ func flattenKubeletConfig(c *containerBeta.NodeKubeletConfig) []map[string]inter
func flattenLinuxNodeConfig(c *containerBeta.LinuxNodeConfig) []map[string]interface{} {
result := []map[string]interface{}{}
if c != nil {
// We are getting the sysctl keys in the Linux format, that is
// with dots in the name, let's replace those with the internal
// TF keys which use underscores.
//
// To get there, let's simply flip our sysAttrsMap and perform
// the lookups.
flipped := func() map[string]string {
m := make(map[string]string)
for k, v := range sysAttrsMap {
m[v] = k
}
return m
}()
m := make(map[string]interface{})
for k, v := range c.Sysctls {
m[strings.ReplaceAll(k, ".", "-")] = v
for sysK, v := range c.Sysctls {
if k, ok := flipped[sysK]; ok {
m[k] = v
}
}
result = append(result, map[string]interface{}{
"sysctls": m,
Expand Down
62 changes: 62 additions & 0 deletions google-beta/resource_container_node_pool.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -769,6 +769,68 @@ func nodePoolUpdate(d *schema.ResourceData, meta interface{}, nodePoolInfo *Node

log.Printf("[INFO] Updated workload_metadata_config for node pool %s", name)
}
if d.HasChange(prefix + "node_config.0.kubelet_config") {
req := &containerBeta.UpdateNodePoolRequest{
NodePoolId: name,
KubeletConfig: expandKubeletConfig(
d.Get(prefix + "node_config.0.kubelet_config")),
}
if req.KubeletConfig == nil {
req.ForceSendFields = []string{"KubeletConfig"}
}
updateF := func() error {
op, err := config.clientContainerBeta.Projects.Locations.Clusters.NodePools.
Update(nodePoolInfo.fullyQualifiedName(name), req).Do()
if err != nil {
return err
}

// Wait until it's updated
return containerOperationWait(config, op,
nodePoolInfo.project,
nodePoolInfo.location,
"updating GKE node pool kubelet_config",
timeout)
}

// Call update serially.
if err := lockedCall(lockKey, updateF); err != nil {
return err
}

log.Printf("[INFO] Updated kubelet_config for node pool %s", name)
}
if d.HasChange(prefix + "node_config.0.linux_node_config") {
req := &containerBeta.UpdateNodePoolRequest{
NodePoolId: name,
LinuxNodeConfig: expandLinuxNodeConfig(
d.Get(prefix + "node_config.0.linux_node_config")),
}
if req.LinuxNodeConfig == nil {
req.ForceSendFields = []string{"LinuxNodeConfig"}
}
updateF := func() error {
op, err := config.clientContainerBeta.Projects.Locations.Clusters.NodePools.
Update(nodePoolInfo.fullyQualifiedName(name), req).Do()
if err != nil {
return err
}

// Wait until it's updated
return containerOperationWait(config, op,
nodePoolInfo.project,
nodePoolInfo.location,
"updating GKE node pool linux_node_config",
timeout)
}

// Call update serially.
if err := lockedCall(lockKey, updateF); err != nil {
return err
}

log.Printf("[INFO] Updated linux_node_config for node pool %s", name)
}

if prefix == "" {
d.SetPartial("node_config")
Expand Down
159 changes: 159 additions & 0 deletions google-beta/resource_container_node_pool_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -238,6 +238,89 @@ func TestAccContainerNodePool_withSandboxConfig(t *testing.T) {
})
}

func TestAccContainerNodePool_withKubeletConfig(t *testing.T) {
t.Parallel()

cluster := fmt.Sprintf("tf-test-cluster-%s", randString(t, 10))
np := fmt.Sprintf("tf-test-np-%s", randString(t, 10))

vcrTest(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders,
CheckDestroy: testAccCheckContainerClusterDestroyProducer(t),
Steps: []resource.TestStep{
{
Config: testAccContainerNodePool_withKubeletConfig(cluster, np, "static"),
Check: resource.ComposeTestCheckFunc(
resource.TestCheckResourceAttr("google_container_node_pool.with_kubelet_config",
"node_config.0.kubelet_config.0.cpu_manager_policy", "static"),
resource.TestCheckResourceAttr("google_container_node_pool.with_kubelet_config",
"node_config.0.kubelet_config.0.cpu_cfs_quota", "true"),
resource.TestCheckResourceAttr("google_container_node_pool.with_kubelet_config",
"node_config.0.kubelet_config.0.cpu_cfs_quota_period", "100us"),
),
},
{
ResourceName: "google_container_node_pool.with_kubelet_config",
ImportState: true,
ImportStateVerify: true,
},
},
})
}

func TestAccContainerNodePool_withInvalidKubeletCpuManagerPolicy(t *testing.T) {
t.Parallel()

cluster := fmt.Sprintf("tf-test-cluster-%s", randString(t, 10))
np := fmt.Sprintf("tf-test-np-%s", randString(t, 10))

vcrTest(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders,
CheckDestroy: testAccCheckContainerClusterDestroyProducer(t),
Steps: []resource.TestStep{
{
Config: testAccContainerNodePool_withKubeletConfig(cluster, np, "dontexist"),
ExpectError: regexp.MustCompile(`.*to be one of \[static default\].*`),
},
},
})
}

func TestAccContainerNodePool_withLinuxNodeConfig(t *testing.T) {
t.Parallel()

cluster := fmt.Sprintf("tf-test-cluster-%s", randString(t, 10))
np := fmt.Sprintf("tf-test-np-%s", randString(t, 10))

vcrTest(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders,
CheckDestroy: testAccCheckContainerClusterDestroyProducer(t),
Steps: []resource.TestStep{
{
Config: testAccContainerNodePool_withLinuxNodeConfig(cluster, np, 100, 128, "10 100 1000", 1),
Check: resource.ComposeTestCheckFunc(
resource.TestCheckResourceAttr("google_container_node_pool.with_kubelet_config",
"node_config.0.linux_node_config.0.sysctls.0.net_core_netdev_max_backlog", "100"),
resource.TestCheckResourceAttr("google_container_node_pool.with_kubelet_config",
"node_config.0.linux_node_config.0.sysctls.0.net_core_somaxconn", "128"),
resource.TestCheckResourceAttr("google_container_node_pool.with_kubelet_config",
"node_config.0.linux_node_config.0.sysctls.0.net_ipv4_tcp_rmem", "10 100 1000"),
resource.TestCheckResourceAttr("google_container_node_pool.with_kubelet_config",
"node_config.0.linux_node_config.0.sysctls.0.net_ipv4_tcp_tw_reuse", "1"),
),
},
{
ResourceName: "google_container_node_pool.with_linux_node_config",
ImportState: true,
ImportStateVerify: true,
},
},
})
}

func TestAccContainerNodePool_withBootDiskKmsKey(t *testing.T) {
t.Parallel()

Expand Down Expand Up @@ -1263,6 +1346,82 @@ resource "google_container_node_pool" "with_sandbox_config" {
`, cluster, np)
}

func testAccContainerNodePool_withKubeletConfig(cluster, np, policy string) string {
return fmt.Sprintf(`
data "google_container_engine_versions" "central1a" {
location = "us-central1-a"
}
resource "google_container_cluster" "cluster" {
name = "%s"
location = "us-central1-a"
initial_node_count = 1
min_master_version = data.google_container_engine_versions.central1a.latest_master_version
}
resource "google_container_node_pool" "with_kubelet_config" {
name = "%s"
location = "us-central1-a"
cluster = google_container_cluster.cluster.name
initial_node_count = 1
node_config {
image_type = "COS_CONTAINERD"
kubelet_config {
cpu_manager_policy = "%s"
cpu_cfs_quota = true
cpu_cfs_quota_period = "100us"
}
oauth_scopes = [
"https://www.googleapis.com/auth/logging.write",
"https://www.googleapis.com/auth/monitoring",
]
}
}
`, cluster, np, policy)
}

func testAccContainerNodePool_withLinuxNodeConfig(cluster, np string, maxBacklog, soMaxConn int, tcpMem string, twReuse int) string {
return fmt.Sprintf(`
data "google_container_engine_versions" "central1a" {
location = "us-central1-a"
}
resource "google_container_cluster" "cluster" {
name = "%s"
location = "us-central1-a"
initial_node_count = 1
min_master_version = data.google_container_engine_versions.central1a.latest_master_version
}
resource "google_container_node_pool" "with_linux_node_config" {
name = "%s"
location = "us-central1-a"
cluster = google_container_cluster.cluster.name
initial_node_count = 1
node_config {
image_type = "COS_CONTAINERD"
linux_node_config {
sysctls {
net_core_netdev_max_backlog = %d
net_core_rmem_max = 100
net_core_wmem_default = 100
net_core_wmem_max = 200
net_core_optmem_max = 100
net_core_somaxconn = %d
net_ipv4_tcp_rmem = "%s"
net_ipv4_tcp_wmem = "%s"
net_ipv4_tcp_tw_reuse = %d
}
}
oauth_scopes = [
"https://www.googleapis.com/auth/logging.write",
"https://www.googleapis.com/auth/monitoring",
]
}
}
`, cluster, np, maxBacklog, soMaxConn, tcpMem, tcpMem, twReuse)
}

func testAccContainerNodePool_withBootDiskKmsKey(project, cluster, np string) string {
return fmt.Sprintf(`
data "google_container_engine_versions" "central1a" {
Expand Down

0 comments on commit 812a880

Please sign in to comment.