packet: Add cpu_manager_policy to workers

This allows a user to choose the cpu manager policy on a worker pool. Possible values are: `none` and `static`. To make this work, kubelet also needs a static allocation of system reserved and kubernetes reserved CPUs to be defined. So this commit also adds the default of 2 cores for each of them only when `cpu_manager_policy` is set. closes: #1337 Signed-off-by: knrt10 <kautilya@kinvolk.io> Co-authored-by: Suraj Deshmukh <suraj@kinvolk.io>
kinvolk · Jul 28, 2021 · 2c38e58 · 2c38e58
1 parent 77629d3
commit 2c38e58
Show file tree

Hide file tree

Showing 7 changed files with 97 additions and 65 deletions.
diff --git a/assets/terraform-modules/packet/flatcar-linux/kubernetes/workers/cl/worker.yaml.tmpl b/assets/terraform-modules/packet/flatcar-linux/kubernetes/workers/cl/worker.yaml.tmpl
@@ -360,6 +360,13 @@ storage:
           apiVersion: kubelet.config.k8s.io/v1beta1
           kind: KubeletConfiguration
           cgroupDriver: "$${docker_cgroup_driver}"
+          %{~ if cpu_manager_policy == "static" ~}
+          cpuManagerPolicy: ${cpu_manager_policy}
+          systemReserved:
+            cpu: ${system_reserved_cpu}
+          kubeReserved:
+            cpu: ${kube_reserved_cpu}
+          %{~ endif ~}
           EOF
     - path: /opt/wait-for-dns
       filesystem: root

diff --git a/assets/terraform-modules/packet/flatcar-linux/kubernetes/workers/variables.tf b/assets/terraform-modules/packet/flatcar-linux/kubernetes/workers/variables.tf
@@ -195,3 +195,21 @@ variable "enable_tls_bootstrap" {
   description = "Enable TLS Bootstrap for Kubelet."
   type        = bool
 }
+
+variable "cpu_manager_policy" {
+  description = "CPU Manager policy to use for the worker pool. Possible values: `none`, `static`."
+  default     = "none"
+  type        = string
+}
+
+variable "kube_reserved_cpu" {
+  description = "CPU cores reserved for the Worker Kubernetes components like kubelet, etc."
+  default     = "500m"
+  type        = string
+}
+
+variable "system_reserved_cpu" {
+  description = "CPU cores reserved for the host services like Docker, sshd, kernel, etc."
+  default     = "1500m"
+  type        = string
+}
diff --git a/assets/terraform-modules/packet/flatcar-linux/kubernetes/workers/workers.tf b/assets/terraform-modules/packet/flatcar-linux/kubernetes/workers/workers.tf
@@ -69,6 +69,9 @@ data "ct_config" "ignitions" {
       cluster_name         = var.cluster_name
       dns_zone             = var.dns_zone
       enable_tls_bootstrap = var.enable_tls_bootstrap
+      cpu_manager_policy   = var.cpu_manager_policy
+      system_reserved_cpu  = var.system_reserved_cpu
+      kube_reserved_cpu    = var.kube_reserved_cpu
     }
   )
   platform = "packet"

diff --git a/docs/configuration-reference/platforms/packet.md b/docs/configuration-reference/platforms/packet.md
diff --git a/pkg/assets/generated_assets.go b/pkg/assets/generated_assets.go
diff --git a/pkg/platform/packet/packet.go b/pkg/platform/packet/packet.go
@@ -46,6 +46,7 @@ const (
 type workerPool struct {
 	Name                  string            `hcl:"pool_name,label"`
 	Count                 int               `hcl:"count"`
+	CPUManagerPolicy      string            `hcl:"cpu_manager_policy,optional"`
 	Facility              string            `hcl:"facility,optional"`
 	DisableBGP            bool              `hcl:"disable_bgp,optional"`
 	IPXEScriptURL         string            `hcl:"ipxe_script_url,optional"`

diff --git a/pkg/platform/packet/template.go b/pkg/platform/packet/template.go
@@ -159,6 +159,8 @@ EOF
 
   cluster_name = "{{$.Config.ClusterName}}"
 
+  cpu_manager_policy = "{{$pool.CPUManagerPolicy}}"
+
   {{- if $pool.Tags }}
   tags = [
       {{- range $key, $value := $pool.Tags }}