packet: Add cpu_manager_policy to workers

This allows a user to choose the cpu manager policy on a worker pool. Possible values are: `none` and `static`. To make this work, kubelet also needs a static allocation of system reserved and kubernetes reserved CPUs to be defined. So this commit also adds the default of 300m cores for kube-reserved-cpu and 1500m cores for system-reserved-cpu when `cpu_manager_policy` is set. closes: #1337 Signed-off-by: knrt10 <kautilya@kinvolk.io> Co-authored-by: Suraj Deshmukh <suraj@kinvolk.io>
kinvolk · Aug 24, 2021 · 7b2fd8a · 7b2fd8a
1 parent bc54c24
commit 7b2fd8a
Show file tree

Hide file tree

Showing 6 changed files with 44 additions and 0 deletions.
diff --git a/assets/terraform-modules/packet/flatcar-linux/kubernetes/workers/cl/worker.yaml.tmpl b/assets/terraform-modules/packet/flatcar-linux/kubernetes/workers/cl/worker.yaml.tmpl
@@ -364,6 +364,13 @@ storage:
           apiVersion: kubelet.config.k8s.io/v1beta1
           kind: KubeletConfiguration
           cgroupDriver: "$${docker_cgroup_driver}"
+          %{~ if cpu_manager_policy == "static" ~}
+          cpuManagerPolicy: ${cpu_manager_policy}
+          systemReserved:
+            cpu: ${system_reserved_cpu}
+          kubeReserved:
+            cpu: ${kube_reserved_cpu}
+          %{~ endif ~}
           EOF
     - path: /opt/wait-for-dns
       filesystem: root

diff --git a/assets/terraform-modules/packet/flatcar-linux/kubernetes/workers/variables.tf b/assets/terraform-modules/packet/flatcar-linux/kubernetes/workers/variables.tf
@@ -195,3 +195,21 @@ variable "enable_tls_bootstrap" {
   description = "Enable TLS Bootstrap for Kubelet."
   type        = bool
 }
+
+variable "cpu_manager_policy" {
+  description = "CPU Manager policy to use for the worker pool. Possible values: `none`, `static`."
+  default     = "none"
+  type        = string
+}
+
+variable "kube_reserved_cpu" {
+  description = "CPU cores reserved for the Worker Kubernetes components like kubelet, etc."
+  default     = "300m"
+  type        = string
+}
+
+variable "system_reserved_cpu" {
+  description = "CPU cores reserved for the host services like Docker, sshd, kernel, etc."
+  default     = "1500m"
+  type        = string
+}
diff --git a/assets/terraform-modules/packet/flatcar-linux/kubernetes/workers/workers.tf b/assets/terraform-modules/packet/flatcar-linux/kubernetes/workers/workers.tf
@@ -69,6 +69,9 @@ data "ct_config" "ignitions" {
       cluster_name         = var.cluster_name
       dns_zone             = var.dns_zone
       enable_tls_bootstrap = var.enable_tls_bootstrap
+      cpu_manager_policy   = var.cpu_manager_policy
+      system_reserved_cpu  = var.system_reserved_cpu
+      kube_reserved_cpu    = var.kube_reserved_cpu
     }
   )
   platform = "packet"

diff --git a/docs/configuration-reference/platforms/equinix-metal.md b/docs/configuration-reference/platforms/equinix-metal.md
@@ -241,6 +241,7 @@ node_type = var.custom_default_worker_type
 | `worker_pool.count`                   | Number of workers in the worker pool. Can be changed afterwards to add or delete workers.                                                                                                                                                                                                                                                 | 1                         | number       | true     |
 | `worker_pool.clc_snippets`            | Flatcar Container Linux Config snippets for nodes in the worker pool.                                                                                                                                                                                                                                                                     | []                        | list(string) | false    |
 | `worker_pool.tags`                    | List of tags that will be propagated to nodes in the worker pool.                                                                                                                                                                                                                                                                         | -                         | map(string)  | false    |
+| `worker_pool.cpu_manager_policy`      | CPU Manager policy to use. Possible values: `none`, `static`.                                                                                                                                                                                                                                                                             | "none"                    | string       | false    |
 | `worker_pool.disable_bgp`             | Disable BGP on nodes. Nodes won't be able to connect to Equinix Metal BGP peers.                                                                                                                                                                                                                                                          | false                     | bool         | false    |
 | `worker_pool.ipxe_script_url`         | Boot via iPXE. Required for arm64.                                                                                                                                                                                                                                                                                                        | -                         | string       | false    |
 | `worker_pool.os_arch`                 | Flatcar Container Linux architecture to install (amd64, arm64).                                                                                                                                                                                                                                                                           | "amd64"                   | string       | false    |

diff --git a/pkg/platform/packet/packet.go b/pkg/platform/packet/packet.go
@@ -46,6 +46,7 @@ const (
 type workerPool struct {
 	Name                  string            `hcl:"pool_name,label"`
 	Count                 int               `hcl:"count"`
+	CPUManagerPolicy      string            `hcl:"cpu_manager_policy,optional"`
 	Facility              string            `hcl:"facility,optional"`
 	DisableBGP            bool              `hcl:"disable_bgp,optional"`
 	IPXEScriptURL         string            `hcl:"ipxe_script_url,optional"`
@@ -513,6 +514,7 @@ func (c *config) checkValidConfig() hcl.Diagnostics {
 	diagnostics = append(diagnostics, c.checkWorkerPoolNamesUnique()...)
 	diagnostics = append(diagnostics, c.checkReservationIDs()...)
 	diagnostics = append(diagnostics, c.validateOSVersion()...)
+	diagnostics = append(diagnostics, c.checkCPUManagerPolicy()...)
 
 	if c.ConntrackMaxPerCore < 0 {
 		diagnostics = append(diagnostics, &hcl.Diagnostic{
@@ -614,6 +616,15 @@ func (c *config) validateOSVersion() hcl.Diagnostics {
 	return diagnostics
 }
 
+func (c *config) checkCPUManagerPolicy() hcl.Diagnostics {
+	wp := make(map[string]string)
+	for _, w := range c.WorkerPools {
+		wp[w.Name] = w.CPUManagerPolicy
+	}
+
+	return platform.CheckCPUManagerPolicy(wp)
+}
+
 // checkEachReservation checks that hardware reservations are in the correct
 // format and, when it will cause problems, that reservation IDs values in this
 // pool are not mixed between using "next-available" and specific UUIDs, as this

diff --git a/pkg/platform/packet/template.go b/pkg/platform/packet/template.go
@@ -159,6 +159,10 @@ EOF
 
   cluster_name = "{{$.Config.ClusterName}}"
 
+  {{- if $pool.CPUManagerPolicy }}
+  cpu_manager_policy = "{{$pool.CPUManagerPolicy}}"
+  {{- end}}
+
   {{- if $pool.Tags }}
   tags = [
       {{- range $key, $value := $pool.Tags }}