From da04a8ceaf4339fd7bc0a26260de10145d31c053 Mon Sep 17 00:00:00 2001 From: Tianhao Geng Date: Fri, 23 Apr 2021 17:08:55 +0000 Subject: [PATCH 1/3] kubelet: add setting for configuring registryPullQPS pass registry-qps argument to kubelet --- README.md | 1 + packages/kubernetes-1.16/kubelet-config | 3 +++ packages/kubernetes-1.17/kubelet-config | 3 +++ packages/kubernetes-1.18/kubelet-config | 3 +++ packages/kubernetes-1.19/kubelet-config | 3 +++ packages/kubernetes-1.20/kubelet-config | 3 +++ sources/models/src/lib.rs | 1 + 7 files changed, 17 insertions(+) diff --git a/README.md b/README.md index cc53e0520cd..6ab58c7064b 100644 --- a/README.md +++ b/README.md @@ -325,6 +325,7 @@ The following settings are optional and allow you to further configure your clus ``` allowed-unsafe-sysctls = ["net.core.somaxconn", "net.ipv4.ip_local_port_range"] ``` +* `settings.kubernetes.registry-qps`: The registry pull QPS. You can also optionally specify static pods for your node with the following settings. Static pods can be particularly useful when running in standalone mode. diff --git a/packages/kubernetes-1.16/kubelet-config b/packages/kubernetes-1.16/kubelet-config index 692395cbf0f..e3f28b49768 100644 --- a/packages/kubernetes-1.16/kubelet-config +++ b/packages/kubernetes-1.16/kubelet-config @@ -40,6 +40,9 @@ evictionHard: {{~#if settings.kubernetes.allowed-unsafe-sysctls}} allowedUnsafeSysctls: {{settings.kubernetes.allowed-unsafe-sysctls}} {{~/if}} +{{~#if settings.kubernetes.registry-qps}} +registryPullQPS: {{settings.kubernetes.registry-qps}} +{{~/if}} kubeReserved: cpu: "{{kube_reserve_cpu settings.kubernetes.kube-reserved.cpu}}" {{~#if settings.kubernetes.kube-reserved.memory}} diff --git a/packages/kubernetes-1.17/kubelet-config b/packages/kubernetes-1.17/kubelet-config index f977392433a..0108c206a49 100644 --- a/packages/kubernetes-1.17/kubelet-config +++ b/packages/kubernetes-1.17/kubelet-config @@ -40,6 +40,9 @@ evictionHard: {{~#if settings.kubernetes.allowed-unsafe-sysctls}} allowedUnsafeSysctls: {{settings.kubernetes.allowed-unsafe-sysctls}} {{~/if}} +{{~#if settings.kubernetes.registry-qps}} +registryPullQPS: {{settings.kubernetes.registry-qps}} +{{~/if}} kubeReserved: cpu: "{{kube_reserve_cpu settings.kubernetes.kube-reserved.cpu}}" {{~#if settings.kubernetes.kube-reserved.memory}} diff --git a/packages/kubernetes-1.18/kubelet-config b/packages/kubernetes-1.18/kubelet-config index f977392433a..0108c206a49 100644 --- a/packages/kubernetes-1.18/kubelet-config +++ b/packages/kubernetes-1.18/kubelet-config @@ -40,6 +40,9 @@ evictionHard: {{~#if settings.kubernetes.allowed-unsafe-sysctls}} allowedUnsafeSysctls: {{settings.kubernetes.allowed-unsafe-sysctls}} {{~/if}} +{{~#if settings.kubernetes.registry-qps}} +registryPullQPS: {{settings.kubernetes.registry-qps}} +{{~/if}} kubeReserved: cpu: "{{kube_reserve_cpu settings.kubernetes.kube-reserved.cpu}}" {{~#if settings.kubernetes.kube-reserved.memory}} diff --git a/packages/kubernetes-1.19/kubelet-config b/packages/kubernetes-1.19/kubelet-config index a6ed4b4d852..b1b168bd202 100644 --- a/packages/kubernetes-1.19/kubelet-config +++ b/packages/kubernetes-1.19/kubelet-config @@ -40,6 +40,9 @@ evictionHard: {{~#if settings.kubernetes.allowed-unsafe-sysctls}} allowedUnsafeSysctls: {{settings.kubernetes.allowed-unsafe-sysctls}} {{~/if}} +{{~#if settings.kubernetes.registry-qps}} +registryPullQPS: {{settings.kubernetes.registry-qps}} +{{~/if}} kubeReserved: cpu: "{{kube_reserve_cpu settings.kubernetes.kube-reserved.cpu}}" {{~#if settings.kubernetes.kube-reserved.memory}} diff --git a/packages/kubernetes-1.20/kubelet-config b/packages/kubernetes-1.20/kubelet-config index 5e95633bd20..2c1b39ed609 100644 --- a/packages/kubernetes-1.20/kubelet-config +++ b/packages/kubernetes-1.20/kubelet-config @@ -40,6 +40,9 @@ evictionHard: {{~#if settings.kubernetes.allowed-unsafe-sysctls}} allowedUnsafeSysctls: {{settings.kubernetes.allowed-unsafe-sysctls}} {{~/if}} +{{~#if settings.kubernetes.registry-qps}} +registryPullQPS: {{settings.kubernetes.registry-qps}} +{{~/if}} kubeReserved: cpu: "{{kube_reserve_cpu settings.kubernetes.kube-reserved.cpu}}" {{~#if settings.kubernetes.kube-reserved.memory}} diff --git a/sources/models/src/lib.rs b/sources/models/src/lib.rs index ef2d395ea21..23186925866 100644 --- a/sources/models/src/lib.rs +++ b/sources/models/src/lib.rs @@ -134,6 +134,7 @@ struct KubernetesSettings { allowed_unsafe_sysctls: Vec, server_tls_bootstrap: bool, cloud_provider: KubernetesCloudProvider, + registry_qps: i32, // Settings where we generate a value based on the runtime environment. The user can specify a // value to override the generated one, but typically would not. From 8d3cc539562d509fb03d82f95e490c60cddc9935 Mon Sep 17 00:00:00 2001 From: Tianhao Geng Date: Sun, 25 Apr 2021 20:26:21 +0000 Subject: [PATCH 2/3] kubelet: add setting for configuring registryBurst pass registry-burst argument to kubelet --- README.md | 1 + packages/kubernetes-1.16/kubelet-config | 3 +++ packages/kubernetes-1.17/kubelet-config | 3 +++ packages/kubernetes-1.18/kubelet-config | 3 +++ packages/kubernetes-1.19/kubelet-config | 3 +++ packages/kubernetes-1.20/kubelet-config | 3 +++ sources/models/src/lib.rs | 1 + 7 files changed, 17 insertions(+) diff --git a/README.md b/README.md index 6ab58c7064b..5294f94dada 100644 --- a/README.md +++ b/README.md @@ -326,6 +326,7 @@ The following settings are optional and allow you to further configure your clus allowed-unsafe-sysctls = ["net.core.somaxconn", "net.ipv4.ip_local_port_range"] ``` * `settings.kubernetes.registry-qps`: The registry pull QPS. +* `settings.kubernetes.registry-burst`: The maximum size of bursty pulls. You can also optionally specify static pods for your node with the following settings. Static pods can be particularly useful when running in standalone mode. diff --git a/packages/kubernetes-1.16/kubelet-config b/packages/kubernetes-1.16/kubelet-config index e3f28b49768..a68481e2146 100644 --- a/packages/kubernetes-1.16/kubelet-config +++ b/packages/kubernetes-1.16/kubelet-config @@ -43,6 +43,9 @@ allowedUnsafeSysctls: {{settings.kubernetes.allowed-unsafe-sysctls}} {{~#if settings.kubernetes.registry-qps}} registryPullQPS: {{settings.kubernetes.registry-qps}} {{~/if}} +{{~#if settings.kubernetes.registry-burst}} +registryBurst: {{settings.kubernetes.registry-burst}} +{{~/if}} kubeReserved: cpu: "{{kube_reserve_cpu settings.kubernetes.kube-reserved.cpu}}" {{~#if settings.kubernetes.kube-reserved.memory}} diff --git a/packages/kubernetes-1.17/kubelet-config b/packages/kubernetes-1.17/kubelet-config index 0108c206a49..e2f92840368 100644 --- a/packages/kubernetes-1.17/kubelet-config +++ b/packages/kubernetes-1.17/kubelet-config @@ -43,6 +43,9 @@ allowedUnsafeSysctls: {{settings.kubernetes.allowed-unsafe-sysctls}} {{~#if settings.kubernetes.registry-qps}} registryPullQPS: {{settings.kubernetes.registry-qps}} {{~/if}} +{{~#if settings.kubernetes.registry-burst}} +registryBurst: {{settings.kubernetes.registry-burst}} +{{~/if}} kubeReserved: cpu: "{{kube_reserve_cpu settings.kubernetes.kube-reserved.cpu}}" {{~#if settings.kubernetes.kube-reserved.memory}} diff --git a/packages/kubernetes-1.18/kubelet-config b/packages/kubernetes-1.18/kubelet-config index 0108c206a49..e2f92840368 100644 --- a/packages/kubernetes-1.18/kubelet-config +++ b/packages/kubernetes-1.18/kubelet-config @@ -43,6 +43,9 @@ allowedUnsafeSysctls: {{settings.kubernetes.allowed-unsafe-sysctls}} {{~#if settings.kubernetes.registry-qps}} registryPullQPS: {{settings.kubernetes.registry-qps}} {{~/if}} +{{~#if settings.kubernetes.registry-burst}} +registryBurst: {{settings.kubernetes.registry-burst}} +{{~/if}} kubeReserved: cpu: "{{kube_reserve_cpu settings.kubernetes.kube-reserved.cpu}}" {{~#if settings.kubernetes.kube-reserved.memory}} diff --git a/packages/kubernetes-1.19/kubelet-config b/packages/kubernetes-1.19/kubelet-config index b1b168bd202..34ab31251d3 100644 --- a/packages/kubernetes-1.19/kubelet-config +++ b/packages/kubernetes-1.19/kubelet-config @@ -43,6 +43,9 @@ allowedUnsafeSysctls: {{settings.kubernetes.allowed-unsafe-sysctls}} {{~#if settings.kubernetes.registry-qps}} registryPullQPS: {{settings.kubernetes.registry-qps}} {{~/if}} +{{~#if settings.kubernetes.registry-burst}} +registryBurst: {{settings.kubernetes.registry-burst}} +{{~/if}} kubeReserved: cpu: "{{kube_reserve_cpu settings.kubernetes.kube-reserved.cpu}}" {{~#if settings.kubernetes.kube-reserved.memory}} diff --git a/packages/kubernetes-1.20/kubelet-config b/packages/kubernetes-1.20/kubelet-config index 2c1b39ed609..67c5a174600 100644 --- a/packages/kubernetes-1.20/kubelet-config +++ b/packages/kubernetes-1.20/kubelet-config @@ -43,6 +43,9 @@ allowedUnsafeSysctls: {{settings.kubernetes.allowed-unsafe-sysctls}} {{~#if settings.kubernetes.registry-qps}} registryPullQPS: {{settings.kubernetes.registry-qps}} {{~/if}} +{{~#if settings.kubernetes.registry-burst}} +registryBurst: {{settings.kubernetes.registry-burst}} +{{~/if}} kubeReserved: cpu: "{{kube_reserve_cpu settings.kubernetes.kube-reserved.cpu}}" {{~#if settings.kubernetes.kube-reserved.memory}} diff --git a/sources/models/src/lib.rs b/sources/models/src/lib.rs index 23186925866..981eba9fa6d 100644 --- a/sources/models/src/lib.rs +++ b/sources/models/src/lib.rs @@ -135,6 +135,7 @@ struct KubernetesSettings { server_tls_bootstrap: bool, cloud_provider: KubernetesCloudProvider, registry_qps: i32, + registry_burst: i32, // Settings where we generate a value based on the runtime environment. The user can specify a // value to override the generated one, but typically would not. From 3b7c693a7ae78c3cda12c1422b498fc4ea80f304 Mon Sep 17 00:00:00 2001 From: Tianhao Geng Date: Mon, 26 Apr 2021 23:28:07 +0000 Subject: [PATCH 3/3] migrations: add migration for adding registry-qps and registry-burst Adds a migration for two new settings `settings.kubernetes.registry-qps` and `settings.kubernetes.registry-burst` --- Release.toml | 1 + sources/Cargo.lock | 7 ++++++ sources/Cargo.toml | 1 + .../Cargo.toml | 12 ++++++++++ .../src/main.rs | 24 +++++++++++++++++++ 5 files changed, 45 insertions(+) create mode 100644 sources/api/migration/migrations/v1.1.0/kubelet-registry-qps-registry-burst/Cargo.toml create mode 100644 sources/api/migration/migrations/v1.1.0/kubelet-registry-qps-registry-burst/src/main.rs diff --git a/Release.toml b/Release.toml index 94c4815d055..b3a97fd056e 100644 --- a/Release.toml +++ b/Release.toml @@ -41,4 +41,5 @@ version = "1.0.8" "(1.0.8, 1.1.0)" = [ "migrate_v1.1.0_kubelet-server-tls-bootstrap.lz4", "migrate_v1.1.0_kubelet-cloud-provider.lz4", + "migrate_v1.1.0_kubelet-registry-qps-registry-burst.lz4", ] diff --git a/sources/Cargo.lock b/sources/Cargo.lock index 532a13f9449..f8341c35a51 100644 --- a/sources/Cargo.lock +++ b/sources/Cargo.lock @@ -1744,6 +1744,13 @@ dependencies = [ "migration-helpers", ] +[[package]] +name = "kubelet-registry-qps-registry-burst" +version = "0.1.0" +dependencies = [ + "migration-helpers", +] + [[package]] name = "kubelet-server-tls-bootstrap" version = "0.1.0" diff --git a/sources/Cargo.toml b/sources/Cargo.toml index e937a9fc52f..76c2b45fa53 100644 --- a/sources/Cargo.toml +++ b/sources/Cargo.toml @@ -55,6 +55,7 @@ members = [ "api/migration/migrations/v1.0.8/add-bootstrap-containers", "api/migration/migrations/v1.1.0/kubelet-server-tls-bootstrap", "api/migration/migrations/v1.1.0/kubelet-cloud-provider", + "api/migration/migrations/v1.1.0/kubelet-registry-qps-registry-burst", "bottlerocket-release", diff --git a/sources/api/migration/migrations/v1.1.0/kubelet-registry-qps-registry-burst/Cargo.toml b/sources/api/migration/migrations/v1.1.0/kubelet-registry-qps-registry-burst/Cargo.toml new file mode 100644 index 00000000000..965099bcd48 --- /dev/null +++ b/sources/api/migration/migrations/v1.1.0/kubelet-registry-qps-registry-burst/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "kubelet-registry-qps-registry-burst" +version = "0.1.0" +authors = ["Tianhao Geng "] +license = "Apache-2.0 OR MIT" +edition = "2018" +publish = false +# Don't rebuild crate just because of changes to README. +exclude = ["README.md"] + +[dependencies] +migration-helpers = { path = "../../../migration-helpers" } diff --git a/sources/api/migration/migrations/v1.1.0/kubelet-registry-qps-registry-burst/src/main.rs b/sources/api/migration/migrations/v1.1.0/kubelet-registry-qps-registry-burst/src/main.rs new file mode 100644 index 00000000000..6fa188d680b --- /dev/null +++ b/sources/api/migration/migrations/v1.1.0/kubelet-registry-qps-registry-burst/src/main.rs @@ -0,0 +1,24 @@ +#![deny(rust_2018_idioms)] + +use migration_helpers::common_migrations::AddSettingsMigration; +use migration_helpers::{migrate, Result}; +use std::process; + +/// We added two new settings for configuring kubelet, `settings.kubernetes.registry-qps` +/// and `settings.kubernetes.registry-burst` +fn run() -> Result<()> { + migrate(AddSettingsMigration(&[ + "settings.kubernetes.registry-qps", + "settings.kubernetes.registry-burst", + ])) +} + +// Returning a Result from main makes it print a Debug representation of the error, but with Snafu +// we have nice Display representations of the error, so we wrap "main" (run) and print any error. +// https://github.com/shepmaster/snafu/issues/110 +fn main() { + if let Err(e) = run() { + eprintln!("{}", e); + process::exit(1); + } +}