aws: Add cpu_manager_policy to workers

This allows a user to choose the cpu manager policy on a worker pool.
Possible values are: `none` and `static`.

To make this work, kubelet also needs a static allocation of system
reserved and kubernetes reserved CPUs to be defined. So this commit also
adds the default of 300m cores for kube-reserved-cpu and 1500m cores for
system-reserved-cpu when `cpu_manager_policy` is set.

Signed-off-by: Suraj Deshmukh <suraj@kinvolk.io>

assets/terraform-modules/aws/flatcar-linux/kubernetes/workers/cl/worker.yaml.tmpl

@@ -140,6 +140,13 @@ storage:
           apiVersion: kubelet.config.k8s.io/v1beta1
           kind: KubeletConfiguration
           cgroupDriver: "$${docker_cgroup_driver}"
+          %{~ if cpu_manager_policy == "static" ~}
+          cpuManagerPolicy: ${cpu_manager_policy}
+          systemReserved:
+            cpu: ${system_reserved_cpu}
+          kubeReserved:
+            cpu: ${kube_reserved_cpu}
+          %{~ endif ~}
           EOF
     - path: /etc/kubernetes/delete-node
       filesystem: root

assets/terraform-modules/aws/flatcar-linux/kubernetes/workers/variables.tf

@@ -173,3 +173,22 @@ variable "enable_csi" {
   type        = bool
   default     = false
 }
+
+
+variable "cpu_manager_policy" {
+  description = "CPU Manager policy to use for the worker pool. Possible values: `none`, `static`."
+  default     = "none"
+  type        = string
+}
+
+variable "kube_reserved_cpu" {
+  description = "CPU cores reserved for the Worker Kubernetes components like kubelet, etc."
+  default     = "300m"
+  type        = string
+}
+
+variable "system_reserved_cpu" {
+  description = "CPU cores reserved for the host services like Docker, sshd, kernel, etc."
+  default     = "1500m"
+  type        = string
+}

assets/terraform-modules/aws/flatcar-linux/kubernetes/workers/workers.tf

@@ -160,6 +160,10 @@ data "ct_config" "worker-ignition" {
     node_labels            = merge({ "node.kubernetes.io/node" = "" }, var.labels)
     taints                 = var.taints
     enable_tls_bootstrap   = var.enable_tls_bootstrap
+    cpu_manager_policy     = var.cpu_manager_policy
+    system_reserved_cpu    = var.system_reserved_cpu
+    kube_reserved_cpu      = var.kube_reserved_cpu
+
   })
   pretty_print = false
   snippets     = var.clc_snippets

pkg/platform/aws/aws.go

@@ -31,23 +31,24 @@ import (
 )
 
 type workerPool struct {
-	Name         string            `hcl:"pool_name,label"`
-	Count        int               `hcl:"count"`
-	SSHPubKeys   []string          `hcl:"ssh_pubkeys"`
-	InstanceType string            `hcl:"instance_type,optional"`
-	OSChannel    string            `hcl:"os_channel,optional"`
-	OSVersion    string            `hcl:"os_version,optional"`
-	Labels       map[string]string `hcl:"labels,optional"`
-	Taints       map[string]string `hcl:"taints,optional"`
-	DiskSize     int               `hcl:"disk_size,optional"`
-	DiskType     string            `hcl:"disk_type,optional"`
-	DiskIOPS     int               `hcl:"disk_iops,optional"`
-	SpotPrice    string            `hcl:"spot_price,optional"`
-	TargetGroups []string          `hcl:"target_groups,optional"`
-	CLCSnippets  []string          `hcl:"clc_snippets,optional"`
-	Tags         map[string]string `hcl:"tags,optional"`
-	LBHTTPPort   int               `hcl:"lb_http_port,optional"`
-	LBHTTPSPort  int               `hcl:"lb_https_port,optional"`
+	Name             string            `hcl:"pool_name,label"`
+	Count            int               `hcl:"count"`
+	CPUManagerPolicy string            `hcl:"cpu_manager_policy,optional"`
+	SSHPubKeys       []string          `hcl:"ssh_pubkeys"`
+	InstanceType     string            `hcl:"instance_type,optional"`
+	OSChannel        string            `hcl:"os_channel,optional"`
+	OSVersion        string            `hcl:"os_version,optional"`
+	Labels           map[string]string `hcl:"labels,optional"`
+	Taints           map[string]string `hcl:"taints,optional"`
+	DiskSize         int               `hcl:"disk_size,optional"`
+	DiskType         string            `hcl:"disk_type,optional"`
+	DiskIOPS         int               `hcl:"disk_iops,optional"`
+	SpotPrice        string            `hcl:"spot_price,optional"`
+	TargetGroups     []string          `hcl:"target_groups,optional"`
+	CLCSnippets      []string          `hcl:"clc_snippets,optional"`
+	Tags             map[string]string `hcl:"tags,optional"`
+	LBHTTPPort       int               `hcl:"lb_http_port,optional"`
+	LBHTTPSPort      int               `hcl:"lb_https_port,optional"`
 }
 
 type config struct {
@@ -281,6 +282,7 @@ func (c *config) checkValidConfig() hcl.Diagnostics {
 	diagnostics = append(diagnostics, c.checkWorkerPoolNamesUnique()...)
 	diagnostics = append(diagnostics, c.checkNameSizes()...)
 	diagnostics = append(diagnostics, c.checkLBPortsUnique()...)
+	diagnostics = append(diagnostics, c.checkCPUManagerPolicy()...)
 
 	if c.ConntrackMaxPerCore < 0 {
 		diagnostics = append(diagnostics, &hcl.Diagnostic{
@@ -410,3 +412,22 @@ func (c *config) checkWorkerPoolNamesUnique() hcl.Diagnostics {
 
 	return diagnostics
 }
+
+func (c *config) checkCPUManagerPolicy() hcl.Diagnostics {
+	var diagnostics hcl.Diagnostics
+
+	for _, w := range c.WorkerPools {
+		switch w.CPUManagerPolicy {
+		case "", "none", "static":
+			continue
+		default:
+			diagnostics = append(diagnostics, &hcl.Diagnostic{
+				Severity: hcl.DiagError,
+				Summary:  "invalid cpu_manager_policy",
+				Detail:   fmt.Sprintf("Worker pool '%s' has invalid cpu_manager_policy '%s'", w.Name, w.CPUManagerPolicy),
+			})
+		}
+	}
+
+	return diagnostics
+}

pkg/platform/aws/aws_internal_test.go

@@ -15,6 +15,7 @@
 package aws
 
 import (
+	"reflect"
 	"testing"
 
 	"github.com/hashicorp/hcl/v2"
@@ -222,3 +223,64 @@ func TestConfigurationIsValidWhen(t *testing.T) {
 		})
 	}
 }
+
+func Test_config_checkCPUManagerPolicy(t *testing.T) {
+	tests := []struct {
+		name        string
+		WorkerPools []workerPool
+		want        hcl.Diagnostics
+	}{
+		{
+			name: "valid_cpu_manager_policy_values",
+			WorkerPools: []workerPool{
+				{CPUManagerPolicy: ""},
+				{CPUManagerPolicy: "static"},
+				{CPUManagerPolicy: "none"},
+			},
+			want: nil,
+		},
+		{
+			name: "invalid_cpu_manager_policy_value",
+			WorkerPools: []workerPool{
+				{CPUManagerPolicy: ""},
+				{Name: "foobar", CPUManagerPolicy: "foobar"},
+			},
+			want: hcl.Diagnostics{&hcl.Diagnostic{
+				Severity: hcl.DiagError,
+				Summary:  "invalid cpu_manager_policy",
+				Detail:   "Worker pool 'foobar' has invalid cpu_manager_policy 'foobar'",
+			}},
+		},
+		{
+			name: "all_invalid_values",
+			WorkerPools: []workerPool{
+				{Name: "foo", CPUManagerPolicy: "foo"},
+				{Name: "bar", CPUManagerPolicy: "bar"},
+			},
+			want: hcl.Diagnostics{
+				&hcl.Diagnostic{
+					Severity: hcl.DiagError,
+					Summary:  "invalid cpu_manager_policy",
+					Detail:   "Worker pool 'foo' has invalid cpu_manager_policy 'foo'",
+				},
+				&hcl.Diagnostic{
+					Severity: hcl.DiagError,
+					Summary:  "invalid cpu_manager_policy",
+					Detail:   "Worker pool 'bar' has invalid cpu_manager_policy 'bar'",
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			c := &config{
+				WorkerPools: tt.WorkerPools,
+			}
+
+			if got := c.checkCPUManagerPolicy(); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("config.checkCPUManagerPolicy() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pkg/platform/aws/template.go

@@ -198,6 +198,10 @@ module "worker-pool-{{ $index }}" {
   clc_snippets          = {{ (index $.WorkerpoolCfg $index "clc_snippets") }}
   {{- end }}
 
+  {{- if $pool.CPUManagerPolicy }}
+  cpu_manager_policy = "{{$pool.CPUManagerPolicy}}"
+  {{- end}}
+
   {{- if $pool.Tags }}
   tags                  = {{ index (index $.WorkerpoolCfg $index) "tags" }}
   {{- end }}