Merge pull request #1606 from gthao313/kubelet-settings

kubelet: add setting for configuring systemReserved
bottlerocket-os · Jun 9, 2021 · 35723e8 · 35723e8
2 parents 06e1634 + 8e1314a
commit 35723e8
Show file tree

Hide file tree

Showing 12 changed files with 83 additions and 0 deletions.
diff --git a/README.md b/README.md
@@ -340,6 +340,14 @@ The following settings are optional and allow you to further configure your clus
     ```
     allowed-unsafe-sysctls = ["net.core.somaxconn", "net.ipv4.ip_local_port_range"]
     ```
+* `settings.kubernetes.system-reserved`: Resources reserved for system components.
+  * Example user data for setting up system reserved:
+    ```
+    [settings.kubernetes.system-reserved]
+    cpu = "10m"
+    memory = "100Mi"
+    ephemeral-storage= "1Gi"
+    ```
 * `settings.kubernetes.registry-qps`: The registry pull QPS.
 * `settings.kubernetes.registry-burst`: The maximum size of bursty pulls.
 * `settings.kubernetes.event-qps`: The maximum event creations per second.

diff --git a/Release.toml b/Release.toml
@@ -50,4 +50,5 @@ version = "1.1.1"
 "(1.1.0, 1.1.1)" = []
 "(1.1.1, 1.1.2)" = [
     "migrate_v1.1.2_kubelet-container-log.lz4",
+    "migrate_v1.1.2_kubelet-system-reserved.lz4",
 ]
diff --git a/packages/kubernetes-1.16/kubelet-config b/packages/kubernetes-1.16/kubelet-config
@@ -68,6 +68,12 @@ kubeReserved:
   {{~/if}}
   {{~/if}}
   ephemeral-storage: "{{default "1Gi" settings.kubernetes.kube-reserved.ephemeral-storage}}"
+{{~#if settings.kubernetes.system-reserved}}
+systemReserved:
+  {{~#each settings.kubernetes.system-reserved}}
+  {{@key}}: "{{this}}"
+  {{~/each}}
+{{~/if}}
 resolvConf: "/etc/resolv.conf"
 hairpinMode: hairpin-veth
 readOnlyPort: 0

diff --git a/packages/kubernetes-1.17/kubelet-config b/packages/kubernetes-1.17/kubelet-config
@@ -68,6 +68,12 @@ kubeReserved:
   {{~/if}}
   {{~/if}}
   ephemeral-storage: "{{default "1Gi" settings.kubernetes.kube-reserved.ephemeral-storage}}"
+{{~#if settings.kubernetes.system-reserved}}
+systemReserved:
+  {{~#each settings.kubernetes.system-reserved}}
+  {{@key}}: "{{this}}"
+  {{~/each}}
+{{~/if}}
 resolvConf: "/etc/resolv.conf"
 hairpinMode: hairpin-veth
 readOnlyPort: 0

diff --git a/packages/kubernetes-1.18/kubelet-config b/packages/kubernetes-1.18/kubelet-config
@@ -68,6 +68,12 @@ kubeReserved:
   {{~/if}}
   {{~/if}}
   ephemeral-storage: "{{default "1Gi" settings.kubernetes.kube-reserved.ephemeral-storage}}"
+{{~#if settings.kubernetes.system-reserved}}
+systemReserved:
+  {{~#each settings.kubernetes.system-reserved}}
+  {{@key}}: "{{this}}"
+  {{~/each}}
+{{~/if}}
 resolvConf: "/etc/resolv.conf"
 hairpinMode: hairpin-veth
 readOnlyPort: 0

diff --git a/packages/kubernetes-1.19/kubelet-config b/packages/kubernetes-1.19/kubelet-config
@@ -68,6 +68,12 @@ kubeReserved:
   {{~/if}}
   {{~/if}}
   ephemeral-storage: "{{default "1Gi" settings.kubernetes.kube-reserved.ephemeral-storage}}"
+{{~#if settings.kubernetes.system-reserved}}
+systemReserved:
+  {{~#each settings.kubernetes.system-reserved}}
+  {{@key}}: "{{this}}"
+  {{~/each}}
+{{~/if}}
 resolvConf: "/etc/resolv.conf"
 hairpinMode: hairpin-veth
 readOnlyPort: 0

diff --git a/packages/kubernetes-1.20/kubelet-config b/packages/kubernetes-1.20/kubelet-config
@@ -68,6 +68,12 @@ kubeReserved:
   {{~/if}}
   {{~/if}}
   ephemeral-storage: "{{default "1Gi" settings.kubernetes.kube-reserved.ephemeral-storage}}"
+{{~#if settings.kubernetes.system-reserved}}
+systemReserved:
+  {{~#each settings.kubernetes.system-reserved}}
+  {{@key}}: "{{this}}"
+  {{~/each}}
+{{~/if}}
 cpuManagerPolicy: "static"
 resolvConf: "/etc/resolv.conf"
 hairpinMode: hairpin-veth

diff --git a/sources/Cargo.lock b/sources/Cargo.lock
diff --git a/sources/Cargo.toml b/sources/Cargo.toml
@@ -32,6 +32,7 @@ members = [
     "api/migration/migrations/v1.1.0/schnauzer-paws",
     "api/migration/migrations/v1.1.0/kubelet-kube-api-qps-kube-api-burst",
     "api/migration/migrations/v1.1.2/kubelet-container-log",
+    "api/migration/migrations/v1.1.2/kubelet-system-reserved",
 
     "bottlerocket-release",
 

diff --git a/sources/api/migration/migrations/v1.1.2/kubelet-system-reserved/Cargo.toml b/sources/api/migration/migrations/v1.1.2/kubelet-system-reserved/Cargo.toml
@@ -0,0 +1,13 @@
+[package]
+name = "kubelet-system-reserved"
+version = "0.1.0"
+authors = ["Tianhao Geng <tianhg@amazon.com>"]
+license = "Apache-2.0 OR MIT"
+edition = "2018"
+publish = false
+# Don't rebuild crate just because of changes to README.
+exclude = ["README.md"]
+
+[dependencies]
+migration-helpers = { path = "../../../migration-helpers" }
+
diff --git a/sources/api/migration/migrations/v1.1.2/kubelet-system-reserved/src/main.rs b/sources/api/migration/migrations/v1.1.2/kubelet-system-reserved/src/main.rs
@@ -0,0 +1,22 @@
+#![deny(rust_2018_idioms)]
+
+use migration_helpers::common_migrations::AddPrefixesMigration;
+use migration_helpers::{migrate, Result};
+use std::process;
+
+/// We added a new settings for configuring kubelet, `settings.kubernetes.system-reserved`
+fn run() -> Result<()> {
+    migrate(AddPrefixesMigration(vec![
+        "settings.kubernetes.system-reserved",
+    ]))
+}
+
+// Returning a Result from main makes it print a Debug representation of the error, but with Snafu
+// we have nice Display representations of the error, so we wrap "main" (run) and print any error.
+// https://github.com/shepmaster/snafu/issues/110
+fn main() {
+    if let Err(e) = run() {
+        eprintln!("{}", e);
+        process::exit(1);
+    }
+}
diff --git a/sources/models/src/lib.rs b/sources/models/src/lib.rs
@@ -136,6 +136,7 @@ struct KubernetesSettings {
     standalone_mode: bool,
     eviction_hard: HashMap<KubernetesEvictionHardKey, KubernetesThresholdValue>,
     kube_reserved: HashMap<KubernetesReservedResourceKey, KubernetesQuantityValue>,
+    system_reserved: HashMap<KubernetesReservedResourceKey, KubernetesQuantityValue>,
     allowed_unsafe_sysctls: Vec<SingleLineString>,
     server_tls_bootstrap: bool,
     cloud_provider: KubernetesCloudProvider,