diff --git a/README.md b/README.md index 0a44086d583..bfbd1e52c36 100644 --- a/README.md +++ b/README.md @@ -304,6 +304,10 @@ The following settings can be optionally set to customize the node labels and ta The following settings are optional and allow you to further configure your cluster. * `settings.kubernetes.cluster-domain`: The DNS domain for this cluster, allowing all Kubernetes-run containers to search this domain before the host's search domains. Defaults to `cluster.local`. +You can also optionally specify static pods for your node with the following settings. +* `settings.kubernetes.static-pods..manifest`: A base64-encoded pod manifest. +* `settings.kubernetes.static-pods..enabled`: Whether the static pod is enabled. + The following settings are set for you automatically by [pluto](sources/api/) based on runtime instance information, but you can override them if you know what you're doing! * `settings.kubernetes.max-pods`: The maximum number of pods that can be scheduled on this node (limited by number of available IPv4 addresses) * `settings.kubernetes.cluster-dns-ip`: The CIDR block of the primary network interface. diff --git a/Release.toml b/Release.toml index 317c3971ae0..01e66888a45 100644 --- a/Release.toml +++ b/Release.toml @@ -20,5 +20,5 @@ version = "1.0.5" "migrate_v1.0.5_add-proxy-restart.lz4", "migrate_v1.0.5_add-proxy-services.lz4" ] -"(1.0.5, 1.0.6)" = ["migrate_v1.0.6_metricdog-init.lz4"] +"(1.0.5, 1.0.6)" = ["migrate_v1.0.6_metricdog-init.lz4", "migrate_v1.0.6_add-static-pods.lz4"] diff --git a/packages/kubernetes-1.15/kubelet-config b/packages/kubernetes-1.15/kubelet-config index a579a9af5d4..709a766bd57 100644 --- a/packages/kubernetes-1.15/kubelet-config +++ b/packages/kubernetes-1.15/kubelet-config @@ -31,3 +31,4 @@ configMapAndSecretChangeDetectionStrategy: Cache tlsCipherSuites: - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 maxPods: {{default 110 settings.kubernetes.max-pods}} +staticPodPath: "/etc/kubernetes/static-pods/" diff --git a/packages/kubernetes-1.15/kubernetes-1.15.spec b/packages/kubernetes-1.15/kubernetes-1.15.spec index 0c2a71ccc1f..2157954632a 100644 --- a/packages/kubernetes-1.15/kubernetes-1.15.spec +++ b/packages/kubernetes-1.15/kubernetes-1.15.spec @@ -20,6 +20,7 @@ Source2: kubelet-env Source3: kubelet-config Source4: kubelet-kubeconfig Source5: kubernetes-ca-crt +Source6: kubernetes-tmpfiles.conf Source1000: clarify.toml Patch1: 0001-always-set-relevant-variables-for-cross-compiling.patch @@ -79,6 +80,9 @@ install -m 0644 %{S:3} %{buildroot}%{_cross_templatedir}/kubelet-config install -m 0644 %{S:4} %{buildroot}%{_cross_templatedir}/kubelet-kubeconfig install -m 0644 %{S:5} %{buildroot}%{_cross_templatedir}/kubernetes-ca-crt +install -d %{buildroot}%{_cross_tmpfilesdir} +install -p -m 0644 %{S:6} %{buildroot}%{_cross_tmpfilesdir}/kubernetes.conf + %cross_scan_attribution --clarify %{S:1000} go-vendor vendor %files -n %{_cross_os}kubelet-1.15 @@ -92,5 +96,6 @@ install -m 0644 %{S:5} %{buildroot}%{_cross_templatedir}/kubernetes-ca-crt %{_cross_templatedir}/kubelet-config %{_cross_templatedir}/kubelet-kubeconfig %{_cross_templatedir}/kubernetes-ca-crt +%{_cross_tmpfilesdir}/kubernetes.conf %changelog diff --git a/packages/kubernetes-1.15/kubernetes-tmpfiles.conf b/packages/kubernetes-1.15/kubernetes-tmpfiles.conf new file mode 100644 index 00000000000..7673fd892f3 --- /dev/null +++ b/packages/kubernetes-1.15/kubernetes-tmpfiles.conf @@ -0,0 +1,2 @@ +d /etc/kubernetes/static-pods - - - - +L /etc/kubernetes/manifests - - - - static-pods diff --git a/packages/kubernetes-1.16/kubelet-config b/packages/kubernetes-1.16/kubelet-config index a579a9af5d4..709a766bd57 100644 --- a/packages/kubernetes-1.16/kubelet-config +++ b/packages/kubernetes-1.16/kubelet-config @@ -31,3 +31,4 @@ configMapAndSecretChangeDetectionStrategy: Cache tlsCipherSuites: - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 maxPods: {{default 110 settings.kubernetes.max-pods}} +staticPodPath: "/etc/kubernetes/static-pods/" diff --git a/packages/kubernetes-1.16/kubernetes-1.16.spec b/packages/kubernetes-1.16/kubernetes-1.16.spec index 46f3882c050..768b99d05a4 100644 --- a/packages/kubernetes-1.16/kubernetes-1.16.spec +++ b/packages/kubernetes-1.16/kubernetes-1.16.spec @@ -20,6 +20,7 @@ Source2: kubelet-env Source3: kubelet-config Source4: kubelet-kubeconfig Source5: kubernetes-ca-crt +Source6: kubernetes-tmpfiles.conf Source1000: clarify.toml Patch1: 0001-always-set-relevant-variables-for-cross-compiling.patch @@ -75,6 +76,9 @@ install -m 0644 %{S:3} %{buildroot}%{_cross_templatedir}/kubelet-config install -m 0644 %{S:4} %{buildroot}%{_cross_templatedir}/kubelet-kubeconfig install -m 0644 %{S:5} %{buildroot}%{_cross_templatedir}/kubernetes-ca-crt +install -d %{buildroot}%{_cross_tmpfilesdir} +install -p -m 0644 %{S:6} %{buildroot}%{_cross_tmpfilesdir}/kubernetes.conf + %cross_scan_attribution --clarify %{S:1000} go-vendor vendor %files -n %{_cross_os}kubelet-1.16 @@ -88,5 +92,6 @@ install -m 0644 %{S:5} %{buildroot}%{_cross_templatedir}/kubernetes-ca-crt %{_cross_templatedir}/kubelet-config %{_cross_templatedir}/kubelet-kubeconfig %{_cross_templatedir}/kubernetes-ca-crt +%{_cross_tmpfilesdir}/kubernetes.conf %changelog diff --git a/packages/kubernetes-1.16/kubernetes-tmpfiles.conf b/packages/kubernetes-1.16/kubernetes-tmpfiles.conf new file mode 100644 index 00000000000..7673fd892f3 --- /dev/null +++ b/packages/kubernetes-1.16/kubernetes-tmpfiles.conf @@ -0,0 +1,2 @@ +d /etc/kubernetes/static-pods - - - - +L /etc/kubernetes/manifests - - - - static-pods diff --git a/packages/kubernetes-1.17/kubelet-config b/packages/kubernetes-1.17/kubelet-config index 78e295c1c13..81b8fa5c041 100644 --- a/packages/kubernetes-1.17/kubelet-config +++ b/packages/kubernetes-1.17/kubelet-config @@ -32,3 +32,4 @@ configMapAndSecretChangeDetectionStrategy: Cache tlsCipherSuites: - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 maxPods: {{default 110 settings.kubernetes.max-pods}} +staticPodPath: "/etc/kubernetes/static-pods/" diff --git a/packages/kubernetes-1.17/kubernetes-1.17.spec b/packages/kubernetes-1.17/kubernetes-1.17.spec index c513a2a4559..a943a5eaf11 100644 --- a/packages/kubernetes-1.17/kubernetes-1.17.spec +++ b/packages/kubernetes-1.17/kubernetes-1.17.spec @@ -20,6 +20,7 @@ Source2: kubelet-env Source3: kubelet-config Source4: kubelet-kubeconfig Source5: kubernetes-ca-crt +Source6: kubernetes-tmpfiles.conf Source1000: clarify.toml Patch1: 0001-always-set-relevant-variables-for-cross-compiling.patch @@ -75,6 +76,9 @@ install -m 0644 %{S:3} %{buildroot}%{_cross_templatedir}/kubelet-config install -m 0644 %{S:4} %{buildroot}%{_cross_templatedir}/kubelet-kubeconfig install -m 0644 %{S:5} %{buildroot}%{_cross_templatedir}/kubernetes-ca-crt +install -d %{buildroot}%{_cross_tmpfilesdir} +install -p -m 0644 %{S:6} %{buildroot}%{_cross_tmpfilesdir}/kubernetes.conf + %cross_scan_attribution --clarify %{S:1000} go-vendor vendor %files -n %{_cross_os}kubelet-1.17 @@ -88,5 +92,6 @@ install -m 0644 %{S:5} %{buildroot}%{_cross_templatedir}/kubernetes-ca-crt %{_cross_templatedir}/kubelet-config %{_cross_templatedir}/kubelet-kubeconfig %{_cross_templatedir}/kubernetes-ca-crt +%{_cross_tmpfilesdir}/kubernetes.conf %changelog diff --git a/packages/kubernetes-1.17/kubernetes-tmpfiles.conf b/packages/kubernetes-1.17/kubernetes-tmpfiles.conf new file mode 100644 index 00000000000..7673fd892f3 --- /dev/null +++ b/packages/kubernetes-1.17/kubernetes-tmpfiles.conf @@ -0,0 +1,2 @@ +d /etc/kubernetes/static-pods - - - - +L /etc/kubernetes/manifests - - - - static-pods diff --git a/packages/kubernetes-1.18/kubelet-config b/packages/kubernetes-1.18/kubelet-config index 78e295c1c13..81b8fa5c041 100644 --- a/packages/kubernetes-1.18/kubelet-config +++ b/packages/kubernetes-1.18/kubelet-config @@ -32,3 +32,4 @@ configMapAndSecretChangeDetectionStrategy: Cache tlsCipherSuites: - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 maxPods: {{default 110 settings.kubernetes.max-pods}} +staticPodPath: "/etc/kubernetes/static-pods/" diff --git a/packages/kubernetes-1.18/kubernetes-1.18.spec b/packages/kubernetes-1.18/kubernetes-1.18.spec index e0e8a802e57..250ef7ba306 100644 --- a/packages/kubernetes-1.18/kubernetes-1.18.spec +++ b/packages/kubernetes-1.18/kubernetes-1.18.spec @@ -20,6 +20,7 @@ Source2: kubelet-env Source3: kubelet-config Source4: kubelet-kubeconfig Source5: kubernetes-ca-crt +Source6: kubernetes-tmpfiles.conf Source1000: clarify.toml Patch1: 0001-always-set-relevant-variables-for-cross-compiling.patch @@ -72,6 +73,9 @@ install -m 0644 %{S:3} %{buildroot}%{_cross_templatedir}/kubelet-config install -m 0644 %{S:4} %{buildroot}%{_cross_templatedir}/kubelet-kubeconfig install -m 0644 %{S:5} %{buildroot}%{_cross_templatedir}/kubernetes-ca-crt +install -d %{buildroot}%{_cross_tmpfilesdir} +install -p -m 0644 %{S:6} %{buildroot}%{_cross_tmpfilesdir}/kubernetes.conf + %cross_scan_attribution --clarify %{S:1000} go-vendor vendor %files -n %{_cross_os}kubelet-1.18 @@ -85,5 +89,6 @@ install -m 0644 %{S:5} %{buildroot}%{_cross_templatedir}/kubernetes-ca-crt %{_cross_templatedir}/kubelet-config %{_cross_templatedir}/kubelet-kubeconfig %{_cross_templatedir}/kubernetes-ca-crt +%{_cross_tmpfilesdir}/kubernetes.conf %changelog diff --git a/packages/kubernetes-1.18/kubernetes-tmpfiles.conf b/packages/kubernetes-1.18/kubernetes-tmpfiles.conf new file mode 100644 index 00000000000..7673fd892f3 --- /dev/null +++ b/packages/kubernetes-1.18/kubernetes-tmpfiles.conf @@ -0,0 +1,2 @@ +d /etc/kubernetes/static-pods - - - - +L /etc/kubernetes/manifests - - - - static-pods diff --git a/packages/kubernetes-1.19/kubelet-config b/packages/kubernetes-1.19/kubelet-config index 3b390066daf..5112b9e6bc2 100644 --- a/packages/kubernetes-1.19/kubelet-config +++ b/packages/kubernetes-1.19/kubelet-config @@ -33,3 +33,4 @@ tlsCipherSuites: - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 volumePluginDir: "/var/lib/kubelet/plugins/volume/exec" maxPods: {{default 110 settings.kubernetes.max-pods}} +staticPodPath: "/etc/kubernetes/static-pods/" diff --git a/packages/kubernetes-1.19/kubernetes-1.19.spec b/packages/kubernetes-1.19/kubernetes-1.19.spec index 47308db1be4..212fcda2d41 100644 --- a/packages/kubernetes-1.19/kubernetes-1.19.spec +++ b/packages/kubernetes-1.19/kubernetes-1.19.spec @@ -20,6 +20,7 @@ Source2: kubelet-env Source3: kubelet-config Source4: kubelet-kubeconfig Source5: kubernetes-ca-crt +Source6: kubernetes-tmpfiles.conf Source1000: clarify.toml Patch1: 0001-always-set-relevant-variables-for-cross-compiling.patch @@ -69,6 +70,9 @@ install -m 0644 %{S:3} %{buildroot}%{_cross_templatedir}/kubelet-config install -m 0644 %{S:4} %{buildroot}%{_cross_templatedir}/kubelet-kubeconfig install -m 0644 %{S:5} %{buildroot}%{_cross_templatedir}/kubernetes-ca-crt +install -d %{buildroot}%{_cross_tmpfilesdir} +install -p -m 0644 %{S:6} %{buildroot}%{_cross_tmpfilesdir}/kubernetes.conf + %cross_scan_attribution --clarify %{S:1000} go-vendor vendor %files -n %{_cross_os}kubelet-1.19 @@ -82,5 +86,6 @@ install -m 0644 %{S:5} %{buildroot}%{_cross_templatedir}/kubernetes-ca-crt %{_cross_templatedir}/kubelet-config %{_cross_templatedir}/kubelet-kubeconfig %{_cross_templatedir}/kubernetes-ca-crt +%{_cross_tmpfilesdir}/kubernetes.conf %changelog diff --git a/packages/kubernetes-1.19/kubernetes-tmpfiles.conf b/packages/kubernetes-1.19/kubernetes-tmpfiles.conf new file mode 100644 index 00000000000..7673fd892f3 --- /dev/null +++ b/packages/kubernetes-1.19/kubernetes-tmpfiles.conf @@ -0,0 +1,2 @@ +d /etc/kubernetes/static-pods - - - - +L /etc/kubernetes/manifests - - - - static-pods diff --git a/packages/os/os.spec b/packages/os/os.spec index d93aaede76e..3fccef853f1 100644 --- a/packages/os/os.spec +++ b/packages/os/os.spec @@ -1,4 +1,5 @@ %global _cross_first_party 1 +%global _is_k8s_variant %(if echo %{_cross_variant} | grep -q "k8s"; then echo 1; else echo 0; fi) %undefine _debugsource_packages Name: %{_cross_os}os @@ -73,7 +74,9 @@ Requires: %{_cross_os}apiserver = %{version}-%{release} Summary: Updates settings dynamically based on user-specified generators Requires: %{_cross_os}apiserver = %{version}-%{release} Requires: %{_cross_os}schnauzer = %{version}-%{release} +%if %{_is_k8s_variant} Requires: %{_cross_os}pluto = %{version}-%{release} +%endif Requires: %{_cross_os}bork = %{version}-%{release} %description -n %{_cross_os}sundog %{summary}. @@ -94,11 +97,6 @@ Summary: Setting generator for templated settings values. %description -n %{_cross_os}schnauzer %{summary}. -%package -n %{_cross_os}pluto -Summary: Dynamic setting generator for kubernetes -%description -n %{_cross_os}pluto -%{summary}. - %package -n %{_cross_os}thar-be-settings Summary: Applies changed settings to a Bottlerocket system Requires: %{_cross_os}apiserver = %{version}-%{release} @@ -182,6 +180,19 @@ Summary: Settings generator for ECS %{summary}. %endif +%if %{_is_k8s_variant} +%package -n %{_cross_os}pluto +Summary: Dynamic setting generator for kubernetes +%description -n %{_cross_os}pluto +%{summary}. + +%package -n %{_cross_os}static-pods +Summary: Manages user-defined K8S static pods +Requires: %{_cross_os}apiserver = %{version}-%{release} +%description -n %{_cross_os}static-pods +%{summary}. +%endif + %prep %setup -T -c %cargo_prep @@ -194,7 +205,6 @@ mkdir bin -p netdog \ -p sundog \ -p schnauzer \ - -p pluto \ -p bork \ -p thar-be-settings \ -p thar-be-updates \ @@ -212,6 +222,10 @@ mkdir bin -p corndog \ %if "%{_cross_variant}" == "aws-ecs-1" -p ecs-settings-applier \ +%endif +%if %{_is_k8s_variant} + -p pluto \ + -p static-pods \ %endif %{nil} @@ -234,7 +248,7 @@ done install -d %{buildroot}%{_cross_bindir} for p in \ apiserver \ - early-boot-config netdog sundog schnauzer pluto bork corndog \ + early-boot-config netdog sundog schnauzer bork corndog \ thar-be-settings thar-be-updates servicedog host-containers \ storewolf settings-committer \ migrator \ @@ -243,6 +257,9 @@ for p in \ %if "%{_cross_variant}" == "aws-ecs-1" ecs-settings-applier \ %endif +%if %{_is_k8s_variant} + pluto static-pods \ +%endif ; do install -p -m 0755 ${HOME}/.cache/%{__cargo_target}/release/${p} %{buildroot}%{_cross_bindir} done @@ -277,8 +294,10 @@ install -d %{buildroot}%{_cross_datadir}/bottlerocket install -d %{buildroot}%{_cross_sysusersdir} install -p -m 0644 %{S:2} %{buildroot}%{_cross_sysusersdir}/api.conf +%if %{_is_k8s_variant} install -d %{buildroot}%{_cross_datadir}/eks install -p -m 0644 %{S:3} %{buildroot}%{_cross_datadir}/eks +%endif install -d %{buildroot}%{_cross_datadir}/updog install -p -m 0644 %{_cross_repo_root_json} %{buildroot}%{_cross_datadir}/updog @@ -332,11 +351,6 @@ install -p -m 0644 %{S:300} %{buildroot}%{_cross_udevrulesdir}/80-ephemeral-stor %files -n %{_cross_os}schnauzer %{_cross_bindir}/schnauzer -%files -n %{_cross_os}pluto -%{_cross_bindir}/pluto -%dir %{_cross_datadir}/eks -%{_cross_datadir}/eks/eni-max-pods - %files -n %{_cross_os}bork %{_cross_bindir}/bork @@ -403,4 +417,14 @@ install -p -m 0644 %{S:300} %{buildroot}%{_cross_udevrulesdir}/80-ephemeral-stor %{_cross_bindir}/ecs-settings-applier %endif +%if %{_is_k8s_variant} +%files -n %{_cross_os}pluto +%{_cross_bindir}/pluto +%dir %{_cross_datadir}/eks +%{_cross_datadir}/eks/eni-max-pods + +%files -n %{_cross_os}static-pods +%{_cross_bindir}/static-pods +%endif + %changelog diff --git a/packages/release/release.spec b/packages/release/release.spec index ad4bfc8b5cc..a08b034540f 100644 --- a/packages/release/release.spec +++ b/packages/release/release.spec @@ -63,7 +63,6 @@ Requires: %{_cross_os}selinux-policy Requires: %{_cross_os}policycoreutils Requires: %{_cross_os}signpost Requires: %{_cross_os}sundog -Requires: %{_cross_os}pluto Requires: %{_cross_os}storewolf Requires: %{_cross_os}host-containers Requires: %{_cross_os}settings-committer diff --git a/sources/Cargo.lock b/sources/Cargo.lock index 90261674de4..5e4dfac2ef9 100644 --- a/sources/Cargo.lock +++ b/sources/Cargo.lock @@ -301,6 +301,13 @@ dependencies = [ "migration-helpers", ] +[[package]] +name = "add-static-pods" +version = "0.1.0" +dependencies = [ + "migration-helpers", +] + [[package]] name = "add-sysctl" version = "0.1.0" @@ -2894,6 +2901,22 @@ dependencies = [ "version_check", ] +[[package]] +name = "static-pods" +version = "0.1.0" +dependencies = [ + "base64 0.13.0", + "cargo-readme", + "log", + "models", + "schnauzer", + "serde", + "serde_json", + "simplelog", + "snafu", + "tokio", +] + [[package]] name = "stdweb" version = "0.4.20" diff --git a/sources/Cargo.toml b/sources/Cargo.toml index 3fe3710d69f..f9219de9028 100644 --- a/sources/Cargo.toml +++ b/sources/Cargo.toml @@ -13,6 +13,7 @@ members = [ "api/pluto", "api/servicedog", "api/host-containers", + "api/static-pods", "api/storewolf", "api/thar-be-settings", "api/thar-be-updates", @@ -38,6 +39,7 @@ members = [ "api/migration/migrations/v1.0.5/add-proxy-restart", "api/migration/migrations/v1.0.5/add-proxy-services", "api/migration/migrations/v1.0.6/metricdog-init", + "api/migration/migrations/v1.0.6/add-static-pods", "bottlerocket-release", diff --git a/sources/api/migration/migrations/v1.0.6/add-static-pods/Cargo.toml b/sources/api/migration/migrations/v1.0.6/add-static-pods/Cargo.toml new file mode 100644 index 00000000000..a258f278a0f --- /dev/null +++ b/sources/api/migration/migrations/v1.0.6/add-static-pods/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "add-static-pods" +version = "0.1.0" +authors = ["Erikson Tung "] +license = "Apache-2.0 OR MIT" +edition = "2018" +publish = false +# Don't rebuild crate just because of changes to README. +exclude = ["README.md"] + +[dependencies] +migration-helpers = { path = "../../../migration-helpers" } diff --git a/sources/api/migration/migrations/v1.0.6/add-static-pods/src/main.rs b/sources/api/migration/migrations/v1.0.6/add-static-pods/src/main.rs new file mode 100644 index 00000000000..a1a803b4050 --- /dev/null +++ b/sources/api/migration/migrations/v1.0.6/add-static-pods/src/main.rs @@ -0,0 +1,24 @@ +#![deny(rust_2018_idioms)] + +use migration_helpers::common_migrations::AddPrefixesMigration; +use migration_helpers::{migrate, Result}; +use std::process; + +/// We added new settings for defining k8s static pods. +/// Remove `settings.kubernetes.static-pods`, `services.static-pods` prefixes when we downgrade. +fn run() -> Result<()> { + migrate(AddPrefixesMigration(vec![ + "settings.kubernetes.static-pods", + "services.static-pods", + ])) +} + +// Returning a Result from main makes it print a Debug representation of the error, but with Snafu +// we have nice Display representations of the error, so we wrap "main" (run) and print any error. +// https://github.com/shepmaster/snafu/issues/110 +fn main() { + if let Err(e) = run() { + eprintln!("{}", e); + process::exit(1); + } +} diff --git a/sources/api/static-pods/Cargo.toml b/sources/api/static-pods/Cargo.toml new file mode 100644 index 00000000000..902337f42e0 --- /dev/null +++ b/sources/api/static-pods/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "static-pods" +version = "0.1.0" +authors = ["Erikson Tung "] +license = "Apache-2.0 OR MIT" +edition = "2018" +publish = false +build = "build.rs" +# Don't rebuild crate just because of changes to README. +exclude = ["README.md"] + +[dependencies] +base64 = "0.13" +log = "0.4" +models = { path = "../../models" } +schnauzer = { path = "../schnauzer" } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1" +simplelog = "0.9" +snafu = "0.6" +tokio = { version = "0.2", default-features = false, features = ["macros", "rt-threaded"] } +# When we update hyper to 0.14+ which has tokio 1: +#tokio = { version = "1", default-features = false, features = ["macros", "rt-multi-thread", "time"] } + +[build-dependencies] +cargo-readme = "3.1" diff --git a/sources/api/static-pods/README.md b/sources/api/static-pods/README.md new file mode 100644 index 00000000000..c365f518b9b --- /dev/null +++ b/sources/api/static-pods/README.md @@ -0,0 +1,18 @@ +# static-pods + +Current version: 0.1.0 + +## Background + +static-pods ensures static pods are running as defined in settings. + +It queries for all existing static pod settings, then configures the system as follows: +* If the pod is enabled, it creates the manifest file in the pod manifest path that kubelet is + configured to read from and populates the file with the base64-decoded manifest setting value. +* If the pod is enabled and the manifest file already exists, it overwrites the existing manifest + file with the base64-decoded manifest setting value. +* If the pod is disabled, it ensures the manifest file is removed from the pod manifest path. + +## Colophon + +This text was generated using [cargo-readme](https://crates.io/crates/cargo-readme), and includes the rustdoc from `src/static_pods.rs`. \ No newline at end of file diff --git a/sources/api/static-pods/README.tpl b/sources/api/static-pods/README.tpl new file mode 100644 index 00000000000..c470c24fe07 --- /dev/null +++ b/sources/api/static-pods/README.tpl @@ -0,0 +1,9 @@ +# {{crate}} + +Current version: {{version}} + +{{readme}} + +## Colophon + +This text was generated using [cargo-readme](https://crates.io/crates/cargo-readme), and includes the rustdoc from `src/static_pods.rs`. diff --git a/sources/api/static-pods/build.rs b/sources/api/static-pods/build.rs new file mode 100644 index 00000000000..08945fa3e7e --- /dev/null +++ b/sources/api/static-pods/build.rs @@ -0,0 +1,41 @@ +// Automatically generate README.md from rustdoc. + +use std::env; +use std::fs::File; +use std::io::Write; +use std::path::PathBuf; + +fn main() { + // TODO: Replace this approach when the build system supports ideas like "variant + // tags": https://github.com/bottlerocket-os/bottlerocket/issues/1260 + println!("cargo:rerun-if-env-changed=VARIANT"); + if let Ok(variant) = env::var("VARIANT") { + if variant.contains("k8s") { + println!("cargo:rustc-cfg=k8s_variant"); + } + } + + // Check for environment variable "SKIP_README". If it is set, + // skip README generation + if env::var_os("SKIP_README").is_some() { + return; + } + + let mut source = File::open("src/static_pods.rs").unwrap(); + let mut template = File::open("README.tpl").unwrap(); + + let content = cargo_readme::generate_readme( + &PathBuf::from("."), // root + &mut source, // source + Some(&mut template), // template + // The "add x" arguments don't apply when using a template. + true, // add title + false, // add badges + false, // add license + true, // indent headings + ) + .unwrap(); + + let mut readme = File::create("README.md").unwrap(); + readme.write_all(content.as_bytes()).unwrap(); +} diff --git a/sources/api/static-pods/src/main.rs b/sources/api/static-pods/src/main.rs new file mode 100644 index 00000000000..a17ee42517a --- /dev/null +++ b/sources/api/static-pods/src/main.rs @@ -0,0 +1,16 @@ +#![deny(rust_2018_idioms)] + +#[cfg(k8s_variant)] +mod static_pods; +#[cfg(k8s_variant)] +#[macro_use] +extern crate log; + +#[cfg(k8s_variant)] +#[tokio::main] +async fn main() { + static_pods::main().await +} + +#[cfg(not(k8s_variant))] +fn main() {} diff --git a/sources/api/static-pods/src/static_pods.rs b/sources/api/static-pods/src/static_pods.rs new file mode 100644 index 00000000000..5d637fe2bfe --- /dev/null +++ b/sources/api/static-pods/src/static_pods.rs @@ -0,0 +1,281 @@ +/*! +# Background + +static-pods ensures static pods are running as defined in settings. + +It queries for all existing static pod settings, then configures the system as follows: +* If the pod is enabled, it creates the manifest file in the pod manifest path that kubelet is + configured to read from and populates the file with the base64-decoded manifest setting value. +* If the pod is enabled and the manifest file already exists, it overwrites the existing manifest + file with the base64-decoded manifest setting value. +* If the pod is disabled, it ensures the manifest file is removed from the pod manifest path. +*/ + +use simplelog::{Config as LogConfig, LevelFilter, SimpleLogger}; +use snafu::{ensure, OptionExt, ResultExt}; +use std::collections::HashMap; +use std::env; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process; +use std::str::FromStr; + +use model::modeled_types::Identifier; + +// FIXME Get from configuration in the future +const DEFAULT_API_SOCKET: &str = "/run/api.sock"; + +const STATIC_POD_DIR: &str = "/etc/kubernetes/static-pods"; + +type Result = std::result::Result; + +/// Query the API for the currently defined static pods +async fn get_static_pods

(socket_path: P) -> Result>> +where + P: AsRef, +{ + debug!("Requesting settings values"); + let settings = schnauzer::get_settings(socket_path) + .await + .context(error::RetrieveSettings)? + .settings + .context(error::MissingSettings)?; + + Ok(settings + .kubernetes + .context(error::MissingSettings)? + .static_pods) +} + +/// Write out the manifest file to the pod manifest path with a given filename +fn write_manifest_file(name: S1, manifest: S2) -> Result<()> +where + S1: AsRef, + S2: AsRef<[u8]>, +{ + let name = name.as_ref(); + + let dir = Path::new(STATIC_POD_DIR); + fs::create_dir_all(&dir).context(error::Mkdir { dir: &dir })?; + let path = dir.join(name); + // Create the file if it does not exist, completely replace its contents + // if it does (constitutes an update). + fs::write(path, manifest).context(error::ManifestWrite { name })?; + + Ok(()) +} + +/// Deletes the named manifest file if it exists +fn delete_manifest_file(name: S1) -> Result<()> +where + S1: AsRef, +{ + let name = name.as_ref(); + let path = Path::new(STATIC_POD_DIR).join(name); + if path.exists() { + fs::remove_file(path).context(error::ManifestDelete { name })?; + } + + Ok(()) +} + +fn handle_static_pod(name: S, pod_info: &model::StaticPod) -> Result<()> +where + S: AsRef, +{ + // Get basic settings, as retrieved from API. + let name = name.as_ref(); + let enabled = pod_info.enabled.context(error::MissingField { + name, + field: "enabled", + })?; + + if enabled { + let manifest = pod_info.manifest.as_ref().context(error::MissingField { + name, + field: "manifest", + })?; + + let manifest = base64::decode(manifest.as_bytes()).context(error::Base64Decode { + base64_string: manifest.as_ref(), + })?; + + info!("Writing static pod '{}' to '{}'", name, STATIC_POD_DIR); + + // Write the manifest file for this static pod + write_manifest_file(name, manifest)?; + } else { + info!("Removing static pod '{}' from '{}'", name, STATIC_POD_DIR); + + // Delete the manifest file so the static pod no longer runs (disabled) + delete_manifest_file(name)?; + } + + Ok(()) +} + +async fn run() -> Result<()> { + let args = parse_args(env::args())?; + + // SimpleLogger will send errors to stderr and anything less to stdout. + SimpleLogger::init(args.log_level, LogConfig::default()).context(error::Logger)?; + + info!("static-pods started"); + + let mut failed = 0u32; + if let Some(static_pods) = get_static_pods(args.socket_path).await? { + for (name, pod) in static_pods.iter() { + // Continue to handle other static pods if we fail one + if let Err(e) = handle_static_pod(name, pod) { + failed += 1; + error!("Failed to handle static pod '{}': {}", &name, e); + } + } + + ensure!( + failed == 0, + error::ManageStaticPodsFailed { + failed, + tried: static_pods.len() + } + ); + } + + Ok(()) +} + +/// Store the args we receive on the command line +struct Args { + log_level: LevelFilter, + socket_path: PathBuf, +} + +/// Print a usage message in the event a bad arg is passed +fn usage() { + let program_name = env::args().next().unwrap_or_else(|| "program".to_string()); + eprintln!( + r"Usage: {} + [ --socket-path PATH ] + [ --log-level trace|debug|info|warn|error ] + + Socket path defaults to {}", + program_name, DEFAULT_API_SOCKET, + ); +} + +/// Parse the args to the program and return an Args struct +fn parse_args(args: env::Args) -> Result { + let mut log_level = None; + let mut socket_path = None; + + let mut iter = args.skip(1); + while let Some(arg) = iter.next() { + match arg.as_ref() { + "--log-level" => { + let log_level_str = iter.next().ok_or_else(|| error::Error::Usage { + message: "Did not give argument to --log-level".into(), + })?; + log_level = Some(LevelFilter::from_str(&log_level_str).map_err(|_| { + error::Error::Usage { + message: format!("Invalid log level '{}'", log_level_str), + } + })?); + } + + "--socket-path" => { + socket_path = Some( + iter.next() + .ok_or_else(|| error::Error::Usage { + message: "Did not give argument to --socket-path".into(), + })? + .into(), + ) + } + + _ => { + return Err(error::Error::Usage { + message: "unexpected argument".into(), + }) + } + } + } + + Ok(Args { + log_level: log_level.unwrap_or_else(|| LevelFilter::Info), + socket_path: socket_path.unwrap_or_else(|| DEFAULT_API_SOCKET.into()), + }) +} + +// Returning a Result from main makes it print a Debug representation of the error, but with Snafu +// we have nice Display representations of the error, so we wrap "main" (run) and print any error. +// https://github.com/shepmaster/snafu/issues/110 +pub(crate) async fn main() -> () { + if let Err(e) = run().await { + match e { + error::Error::Usage { .. } => { + eprintln!("{}", e); + usage(); + process::exit(2); + } + _ => { + eprintln!("{}", e); + process::exit(1); + } + } + } +} + +mod error { + use snafu::Snafu; + use std::path::PathBuf; + + #[derive(Debug, Snafu)] + #[snafu(visibility = "pub(super)")] + pub(super) enum Error { + #[snafu(display("{}", message))] + Usage { message: String }, + + #[snafu(display("Failed to retrieve settings: {}", source))] + RetrieveSettings { source: schnauzer::Error }, + + #[snafu(display("settings.kubernetes.static_pods missing in API response"))] + MissingSettings {}, + + #[snafu(display("Static pod '{}' missing field '{}'", name, field))] + MissingField { name: String, field: String }, + + #[snafu(display("Failed to manage {} of {} static pods", failed, tried))] + ManageStaticPodsFailed { failed: u32, tried: usize }, + + #[snafu(display("Logger setup error: {}", source))] + Logger { source: log::SetLoggerError }, + + #[snafu(display("Unable to base64 decode manifest '{}': '{}'", base64_string, source))] + Base64Decode { + base64_string: String, + source: base64::DecodeError, + }, + + #[snafu(display("Failed to create directory '{}': '{}'", dir.display(), source))] + Mkdir { + dir: PathBuf, + source: std::io::Error, + }, + + #[snafu(display("Failed to write manifest for static pod '{}': {}", name, source))] + ManifestWrite { + name: String, + source: std::io::Error, + }, + + #[snafu(display( + "Failed to delete manifest file for static pod '{}': {}'", + name, + source + ))] + ManifestDelete { + name: String, + source: std::io::Error, + }, + } +} diff --git a/sources/models/src/aws-k8s-1.15/defaults.d/50-aws-k8s.toml b/sources/models/src/aws-k8s-1.15/defaults.d/50-aws-k8s.toml index 491fa9fa1ee..a24885efbb0 100644 --- a/sources/models/src/aws-k8s-1.15/defaults.d/50-aws-k8s.toml +++ b/sources/models/src/aws-k8s-1.15/defaults.d/50-aws-k8s.toml @@ -41,6 +41,13 @@ cluster-domain = "cluster.local" [settings.metrics] service-checks = ["apiserver", "chronyd", "containerd", "host-containerd", "kubelet"] +[services.static-pods] +configuration-files = [] +restart-commands = ["/usr/bin/static-pods"] + +[metadata.settings.kubernetes.static-pods] +affected-services = ["static-pods"] + # Network [metadata.settings.network] affected-services = ["containerd", "kubernetes", "host-containerd"] diff --git a/sources/models/src/lib.rs b/sources/models/src/lib.rs index d30b45f2a4b..79aac47f765 100644 --- a/sources/models/src/lib.rs +++ b/sources/models/src/lib.rs @@ -93,11 +93,18 @@ use std::collections::HashMap; use std::net::Ipv4Addr; use crate::modeled_types::{ - DNSDomain, ECSAgentLogLevel, ECSAttributeKey, ECSAttributeValue, FriendlyVersion, + DNSDomain, ECSAgentLogLevel, ECSAttributeKey, ECSAttributeValue, FriendlyVersion, Identifier, KubernetesClusterName, KubernetesLabelKey, KubernetesLabelValue, KubernetesTaintValue, Lockdown, SingleLineString, SysctlKey, Url, ValidBase64, }; +// Kubernetes static pod manifest settings +#[model] +struct StaticPod { + enabled: bool, + manifest: ValidBase64, +} + // Kubernetes related settings. The dynamic settings are retrieved from // IMDS via Sundog's child "Pluto". #[model] @@ -108,6 +115,7 @@ struct KubernetesSettings { api_server: Url, node_labels: HashMap, node_taints: HashMap, + static_pods: HashMap, // Dynamic settings. max_pods: u32, diff --git a/variants/aws-k8s-1.15/Cargo.toml b/variants/aws-k8s-1.15/Cargo.toml index 304b5c07993..f2d23d3211e 100644 --- a/variants/aws-k8s-1.15/Cargo.toml +++ b/variants/aws-k8s-1.15/Cargo.toml @@ -16,6 +16,8 @@ included-packages = [ "cni-plugins", "kubelet-1.15", "release", + "static-pods", + "pluto" ] [lib] diff --git a/variants/aws-k8s-1.16/Cargo.toml b/variants/aws-k8s-1.16/Cargo.toml index 659be17950c..16ddd2f8c65 100644 --- a/variants/aws-k8s-1.16/Cargo.toml +++ b/variants/aws-k8s-1.16/Cargo.toml @@ -16,6 +16,8 @@ included-packages = [ "cni-plugins", "kubelet-1.16", "release", + "static-pods", + "pluto" ] [lib] diff --git a/variants/aws-k8s-1.17/Cargo.toml b/variants/aws-k8s-1.17/Cargo.toml index 3716075b901..fde8e024263 100644 --- a/variants/aws-k8s-1.17/Cargo.toml +++ b/variants/aws-k8s-1.17/Cargo.toml @@ -16,6 +16,8 @@ included-packages = [ "cni-plugins", "kubelet-1.17", "release", + "static-pods", + "pluto" ] [lib] diff --git a/variants/aws-k8s-1.18/Cargo.toml b/variants/aws-k8s-1.18/Cargo.toml index 958c3b2a869..2e61440fff7 100644 --- a/variants/aws-k8s-1.18/Cargo.toml +++ b/variants/aws-k8s-1.18/Cargo.toml @@ -16,6 +16,8 @@ included-packages = [ "cni-plugins", "kubelet-1.18", "release", + "static-pods", + "pluto" ] [lib] diff --git a/variants/aws-k8s-1.19/Cargo.toml b/variants/aws-k8s-1.19/Cargo.toml index 6de2c309bc0..a64d1c14961 100644 --- a/variants/aws-k8s-1.19/Cargo.toml +++ b/variants/aws-k8s-1.19/Cargo.toml @@ -16,6 +16,8 @@ included-packages = [ "cni-plugins", "kubelet-1.19", "release", + "static-pods", + "pluto" ] [lib]