From 755d4373f991b45a77a85d62993d480110c00bc2 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 28 Dec 2019 19:47:34 -0800 Subject: [PATCH] Inline Container Linux kubelet.service, deprecate kubelet-wrapper * Change kubelet.service on Container Linux nodes to ExecStart Kubelet inline to replace the use of the host OS kubelet-wrapper script * Express rkt run flags and volume mounts in a clear, uniform way to make the Kubelet service easier to audit, manage, and understand * Eliminate reliance on a Container Linux kubelet-wrapper script * Typhoon for Fedora CoreOS developed a kubelet.service that similarly uses an inline ExecStart (except with podman instead of rkt) and a more minimal set of volume mounts. Adopt the volume improvements: * Change Kubelet /etc/kubernetes volume to read-only * Change Kubelet /etc/resolv.conf volume to read-only * Remove unneeded /var/lib/cni volume mount Background: * kubelet-wrapper was added in CoreOS around the time of Kubernetes v1.0 to simplify running a CoreOS-built hyperkube ACI image via rkt-fly. The script defaults are no longer ideal (e.g. rkt's notion of trust dates back to quay.io ACI image serving and signing, which informed the OCI standard images we use today, though they still lack rkt's signing ideas). * Shipping kubelet-wrapper was regretted at CoreOS, but remains in the distro for compatibility. The script is not updated to track hyperkube changes, but it is stable and kubelet.env overrides bridge most gaps * Typhoon Container Linux nodes have used kubelet-wrapper to rkt/rkt-fly run the Kubelet via the official k8s.gcr.io hyperkube image using overrides (new image registry, new image format, restart handling, new mounts, new entrypoint in v1.17). * Observation: Most of what it takes to run a Kubelet container is defined in Typhoon, not in kubelet-wrapper. The wrapper's value is now undermined by having to workaround its dated defaults. Typhoon may be better served defining Kubelet.service explicitly * Typhoon for Fedora CoreOS developed a kubelet.service without the use of a host OS kubelet-wrapper which is both clearer and eliminated some volume mounts --- CHANGES.md | 1 + .../kubernetes/cl/controller.yaml | 56 +++++++++------- .../kubernetes/workers/cl/worker.yaml | 58 ++++++++++------- .../kubernetes/cl/controller.yaml | 58 ++++++++++------- .../kubernetes/workers/cl/worker.yaml | 58 ++++++++++------- .../kubernetes/cl/controller.yaml | 64 +++++++++++-------- .../container-linux/kubernetes/cl/worker.yaml | 64 +++++++++++-------- .../kubernetes/cl/controller.yaml | 56 +++++++++------- .../container-linux/kubernetes/cl/worker.yaml | 56 +++++++++------- .../kubernetes/cl/controller.yaml | 57 ++++++++++------- .../kubernetes/workers/cl/worker.yaml | 57 ++++++++++------- 11 files changed, 342 insertions(+), 243 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 93282a36f..e37a02006 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,7 @@ Notable changes between versions. * Update CoreDNS from v1.6.5 to [v1.6.6](https://coredns.io/2019/12/11/coredns-1.6.6-release/) ([#602](https://github.com/poseidon/typhoon/pull/602)) * Update Calico from v3.10.2 to v3.11.1 ([#604](https://github.com/poseidon/typhoon/pull/604)) +* Inline Kubelet service on Container Linux nodes ([#606](https://github.com/poseidon/typhoon/pull/606)) #### Addons diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index 9240653c8..95fb29bd0 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -50,29 +50,47 @@ systemd: Description=Kubelet via Hyperkube Wants=rpc-statd.service [Service] - EnvironmentFile=/etc/kubernetes/kubelet.env - Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ - --volume=resolv,kind=host,source=/etc/resolv.conf \ - --mount volume=resolv,target=/etc/resolv.conf \ - --volume var-lib-cni,kind=host,source=/var/lib/cni \ - --mount volume=var-lib-cni,target=/var/lib/cni \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --insecure-options=image" Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/lib/coreos/kubelet-wrapper \ + ExecStart=/usr/bin/rkt run \ + --uuid-file-save=/var/cache/kubelet-pod.uuid \ + --stage1-from-dir=stage1-fly.aci \ + --hosts-entry host \ + --insecure-options=image \ + --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ + --mount volume=etc-kubernetes,target=/etc/kubernetes \ + --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ + --mount volume=etc-machine-id,target=/etc/machine-id \ + --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ + --mount volume=etc-os-release,target=/etc/os-release \ + --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ + --mount volume=etc-resolv,target=/etc/resolv.conf \ + --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ + --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ + --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ + --mount volume=lib-modules,target=/lib/modules \ + --volume run,kind=host,source=/run \ + --mount volume=run,target=/run \ + --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ + --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ + --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --mount volume=var-lib-calico,target=/var/lib/calico \ + --volume var-lib-docker,kind=host,source=/var/lib/docker \ + --mount volume=var-lib-docker,target=/var/lib/docker \ + --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ + --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ + --volume var-log,kind=host,source=/var/log \ + --mount volume=var-log,target=/var/log \ + --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ + --mount volume=opt-cni-bin,target=/opt/cni/bin \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ + --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -130,14 +148,6 @@ storage: contents: inline: | ${kubeconfig} - - path: /etc/kubernetes/kubelet.env - filesystem: root - mode: 0644 - contents: - inline: | - KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.17.0 - KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /opt/bootstrap/layout filesystem: root mode: 0544 diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml b/aws/container-linux/kubernetes/workers/cl/worker.yaml index 6a36c8ce8..ba27b8293 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml @@ -25,29 +25,47 @@ systemd: Description=Kubelet via Hyperkube Wants=rpc-statd.service [Service] - EnvironmentFile=/etc/kubernetes/kubelet.env - Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ - --volume=resolv,kind=host,source=/etc/resolv.conf \ - --mount volume=resolv,target=/etc/resolv.conf \ - --volume var-lib-cni,kind=host,source=/var/lib/cni \ - --mount volume=var-lib-cni,target=/var/lib/cni \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --insecure-options=image" Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} - ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d + ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/lib/coreos/kubelet-wrapper \ + ExecStart=/usr/bin/rkt run \ + --uuid-file-save=/var/cache/kubelet-pod.uuid \ + --stage1-from-dir=stage1-fly.aci \ + --hosts-entry host \ + --insecure-options=image \ + --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ + --mount volume=etc-kubernetes,target=/etc/kubernetes \ + --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ + --mount volume=etc-machine-id,target=/etc/machine-id \ + --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ + --mount volume=etc-os-release,target=/etc/os-release \ + --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ + --mount volume=etc-resolv,target=/etc/resolv.conf \ + --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ + --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ + --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ + --mount volume=lib-modules,target=/lib/modules \ + --volume run,kind=host,source=/run \ + --mount volume=run,target=/run \ + --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ + --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ + --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --mount volume=var-lib-calico,target=/var/lib/calico \ + --volume var-lib-docker,kind=host,source=/var/lib/docker \ + --mount volume=var-lib-docker,target=/var/lib/docker \ + --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ + --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ + --volume var-log,kind=host,source=/var/log \ + --mount volume=var-log,target=/var/log \ + --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ + --mount volume=opt-cni-bin,target=/opt/cni/bin \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ + --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -92,14 +110,6 @@ storage: contents: inline: | ${kubeconfig} - - path: /etc/kubernetes/kubelet.env - filesystem: root - mode: 0644 - contents: - inline: | - KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.17.0 - KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index 80f53cd36..236f65e9f 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -50,28 +50,46 @@ systemd: Description=Kubelet via Hyperkube Wants=rpc-statd.service [Service] - EnvironmentFile=/etc/kubernetes/kubelet.env - Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ - --volume=resolv,kind=host,source=/etc/resolv.conf \ - --mount volume=resolv,target=/etc/resolv.conf \ - --volume var-lib-cni,kind=host,source=/var/lib/cni \ - --mount volume=var-lib-cni,target=/var/lib/cni \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --insecure-options=image" - ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d + ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/lib/coreos/kubelet-wrapper \ + ExecStart=/usr/bin/rkt run \ + --uuid-file-save=/var/cache/kubelet-pod.uuid \ + --stage1-from-dir=stage1-fly.aci \ + --hosts-entry host \ + --insecure-options=image \ + --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ + --mount volume=etc-kubernetes,target=/etc/kubernetes \ + --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ + --mount volume=etc-machine-id,target=/etc/machine-id \ + --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ + --mount volume=etc-os-release,target=/etc/os-release \ + --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ + --mount volume=etc-resolv,target=/etc/resolv.conf \ + --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ + --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ + --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ + --mount volume=lib-modules,target=/lib/modules \ + --volume run,kind=host,source=/run \ + --mount volume=run,target=/run \ + --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ + --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ + --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --mount volume=var-lib-calico,target=/var/lib/calico \ + --volume var-lib-docker,kind=host,source=/var/lib/docker \ + --mount volume=var-lib-docker,target=/var/lib/docker \ + --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ + --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ + --volume var-log,kind=host,source=/var/log \ + --mount volume=var-log,target=/var/log \ + --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ + --mount volume=opt-cni-bin,target=/opt/cni/bin \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ + --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -128,14 +146,6 @@ storage: contents: inline: | ${kubeconfig} - - path: /etc/kubernetes/kubelet.env - filesystem: root - mode: 0644 - contents: - inline: | - KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.17.0 - KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /opt/bootstrap/layout filesystem: root mode: 0544 diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml b/azure/container-linux/kubernetes/workers/cl/worker.yaml index d29ecb367..ab0992927 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml @@ -25,28 +25,46 @@ systemd: Description=Kubelet via Hyperkube Wants=rpc-statd.service [Service] - EnvironmentFile=/etc/kubernetes/kubelet.env - Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ - --volume=resolv,kind=host,source=/etc/resolv.conf \ - --mount volume=resolv,target=/etc/resolv.conf \ - --volume var-lib-cni,kind=host,source=/var/lib/cni \ - --mount volume=var-lib-cni,target=/var/lib/cni \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --insecure-options=image" - ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d + ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/lib/coreos/kubelet-wrapper \ + ExecStart=/usr/bin/rkt run \ + --uuid-file-save=/var/cache/kubelet-pod.uuid \ + --stage1-from-dir=stage1-fly.aci \ + --hosts-entry host \ + --insecure-options=image \ + --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ + --mount volume=etc-kubernetes,target=/etc/kubernetes \ + --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ + --mount volume=etc-machine-id,target=/etc/machine-id \ + --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ + --mount volume=etc-os-release,target=/etc/os-release \ + --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ + --mount volume=etc-resolv,target=/etc/resolv.conf \ + --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ + --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ + --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ + --mount volume=lib-modules,target=/lib/modules \ + --volume run,kind=host,source=/run \ + --mount volume=run,target=/run \ + --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ + --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ + --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --mount volume=var-lib-calico,target=/var/lib/calico \ + --volume var-lib-docker,kind=host,source=/var/lib/docker \ + --mount volume=var-lib-docker,target=/var/lib/docker \ + --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ + --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ + --volume var-log,kind=host,source=/var/log \ + --mount volume=var-log,target=/var/log \ + --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ + --mount volume=opt-cni-bin,target=/opt/cni/bin \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ + --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -90,14 +108,6 @@ storage: contents: inline: | ${kubeconfig} - - path: /etc/kubernetes/kubelet.env - filesystem: root - mode: 0644 - contents: - inline: | - KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.17.0 - KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index ddc943f98..42e7d3965 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -58,33 +58,51 @@ systemd: Description=Kubelet via Hyperkube Wants=rpc-statd.service [Service] - EnvironmentFile=/etc/kubernetes/kubelet.env - Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ - --volume=resolv,kind=host,source=/etc/resolv.conf \ - --mount volume=resolv,target=/etc/resolv.conf \ - --volume var-lib-cni,kind=host,source=/var/lib/cni \ - --mount volume=var-lib-cni,target=/var/lib/cni \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --volume iscsiconf,kind=host,source=/etc/iscsi/ \ - --mount volume=iscsiconf,target=/etc/iscsi/ \ - --volume iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ - --mount volume=iscsiadm,target=/sbin/iscsiadm \ - --insecure-options=image" Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/lib/coreos/kubelet-wrapper \ + ExecStart=/usr/bin/rkt run \ + --uuid-file-save=/var/cache/kubelet-pod.uuid \ + --stage1-from-dir=stage1-fly.aci \ + --hosts-entry host \ + --insecure-options=image \ + --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ + --mount volume=etc-kubernetes,target=/etc/kubernetes \ + --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ + --mount volume=etc-machine-id,target=/etc/machine-id \ + --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ + --mount volume=etc-os-release,target=/etc/os-release \ + --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ + --mount volume=etc-resolv,target=/etc/resolv.conf \ + --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ + --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ + --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ + --mount volume=lib-modules,target=/lib/modules \ + --volume run,kind=host,source=/run \ + --mount volume=run,target=/run \ + --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ + --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ + --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --mount volume=var-lib-calico,target=/var/lib/calico \ + --volume var-lib-docker,kind=host,source=/var/lib/docker \ + --mount volume=var-lib-docker,target=/var/lib/docker \ + --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ + --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ + --volume var-log,kind=host,source=/var/log \ + --mount volume=var-log,target=/var/log \ + --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ + --mount volume=opt-cni-bin,target=/opt/cni/bin \ + --volume etc-iscsi,kind=host,source=/etc/iscsi \ + --mount volume=etc-iscsi,target=/etc/iscsi \ + --volume usr-sbin-iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ + --mount volume=usr-sbin-iscsiadm,target=/sbin/iscsiadm \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ + --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -137,14 +155,6 @@ systemd: WantedBy=multi-user.target storage: files: - - path: /etc/kubernetes/kubelet.env - filesystem: root - mode: 0644 - contents: - inline: | - KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.17.0 - KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /etc/hostname filesystem: root mode: 0644 diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml b/bare-metal/container-linux/kubernetes/cl/worker.yaml index efe18ddc9..c2b78e7a3 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml @@ -33,33 +33,51 @@ systemd: Description=Kubelet via Hyperkube Wants=rpc-statd.service [Service] - EnvironmentFile=/etc/kubernetes/kubelet.env - Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ - --volume=resolv,kind=host,source=/etc/resolv.conf \ - --mount volume=resolv,target=/etc/resolv.conf \ - --volume var-lib-cni,kind=host,source=/var/lib/cni \ - --mount volume=var-lib-cni,target=/var/lib/cni \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --volume iscsiconf,kind=host,source=/etc/iscsi/ \ - --mount volume=iscsiconf,target=/etc/iscsi/ \ - --volume iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ - --mount volume=iscsiadm,target=/sbin/iscsiadm \ - --insecure-options=image" Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/lib/coreos/kubelet-wrapper \ + ExecStart=/usr/bin/rkt run \ + --uuid-file-save=/var/cache/kubelet-pod.uuid \ + --stage1-from-dir=stage1-fly.aci \ + --hosts-entry host \ + --insecure-options=image \ + --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ + --mount volume=etc-kubernetes,target=/etc/kubernetes \ + --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ + --mount volume=etc-machine-id,target=/etc/machine-id \ + --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ + --mount volume=etc-os-release,target=/etc/os-release \ + --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ + --mount volume=etc-resolv,target=/etc/resolv.conf \ + --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ + --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ + --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ + --mount volume=lib-modules,target=/lib/modules \ + --volume run,kind=host,source=/run \ + --mount volume=run,target=/run \ + --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ + --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ + --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --mount volume=var-lib-calico,target=/var/lib/calico \ + --volume var-lib-docker,kind=host,source=/var/lib/docker \ + --mount volume=var-lib-docker,target=/var/lib/docker \ + --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ + --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ + --volume var-log,kind=host,source=/var/log \ + --mount volume=var-log,target=/var/log \ + --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ + --mount volume=opt-cni-bin,target=/opt/cni/bin \ + --volume etc-iscsi,kind=host,source=/etc/iscsi \ + --mount volume=etc-iscsi,target=/etc/iscsi \ + --volume usr-sbin-iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ + --mount volume=usr-sbin-iscsiadm,target=/sbin/iscsiadm \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ + --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -85,14 +103,6 @@ systemd: storage: files: - - path: /etc/kubernetes/kubelet.env - filesystem: root - mode: 0644 - contents: - inline: | - KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.17.0 - KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /etc/hostname filesystem: root mode: 0644 diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index d51f24fe8..e654fb454 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -60,29 +60,47 @@ systemd: After=coreos-metadata.service Wants=rpc-statd.service [Service] - EnvironmentFile=/etc/kubernetes/kubelet.env EnvironmentFile=/run/metadata/coreos - Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ - --volume=resolv,kind=host,source=/etc/resolv.conf \ - --mount volume=resolv,target=/etc/resolv.conf \ - --volume var-lib-cni,kind=host,source=/var/lib/cni \ - --mount volume=var-lib-cni,target=/var/lib/cni \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --insecure-options=image" ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/lib/coreos/kubelet-wrapper \ + ExecStart=/usr/bin/rkt run \ + --uuid-file-save=/var/cache/kubelet-pod.uuid \ + --stage1-from-dir=stage1-fly.aci \ + --hosts-entry host \ + --insecure-options=image \ + --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ + --mount volume=etc-kubernetes,target=/etc/kubernetes \ + --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ + --mount volume=etc-machine-id,target=/etc/machine-id \ + --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ + --mount volume=etc-os-release,target=/etc/os-release \ + --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ + --mount volume=etc-resolv,target=/etc/resolv.conf \ + --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ + --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ + --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ + --mount volume=lib-modules,target=/lib/modules \ + --volume run,kind=host,source=/run \ + --mount volume=run,target=/run \ + --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ + --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ + --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --mount volume=var-lib-calico,target=/var/lib/calico \ + --volume var-lib-docker,kind=host,source=/var/lib/docker \ + --mount volume=var-lib-docker,target=/var/lib/docker \ + --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ + --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ + --volume var-log,kind=host,source=/var/log \ + --mount volume=var-log,target=/var/log \ + --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ + --mount volume=opt-cni-bin,target=/opt/cni/bin \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ + --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -134,14 +152,6 @@ systemd: WantedBy=multi-user.target storage: files: - - path: /etc/kubernetes/kubelet.env - filesystem: root - mode: 0644 - contents: - inline: | - KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.17.0 - KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /opt/bootstrap/layout filesystem: root mode: 0544 diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml b/digital-ocean/container-linux/kubernetes/cl/worker.yaml index 4ae4603e5..007014030 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml @@ -35,29 +35,47 @@ systemd: After=coreos-metadata.service Wants=rpc-statd.service [Service] - EnvironmentFile=/etc/kubernetes/kubelet.env EnvironmentFile=/run/metadata/coreos - Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ - --volume=resolv,kind=host,source=/etc/resolv.conf \ - --mount volume=resolv,target=/etc/resolv.conf \ - --volume var-lib-cni,kind=host,source=/var/lib/cni \ - --mount volume=var-lib-cni,target=/var/lib/cni \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --insecure-options=image" ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/lib/coreos/kubelet-wrapper \ + ExecStart=/usr/bin/rkt run \ + --uuid-file-save=/var/cache/kubelet-pod.uuid \ + --stage1-from-dir=stage1-fly.aci \ + --hosts-entry host \ + --insecure-options=image \ + --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ + --mount volume=etc-kubernetes,target=/etc/kubernetes \ + --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ + --mount volume=etc-machine-id,target=/etc/machine-id \ + --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ + --mount volume=etc-os-release,target=/etc/os-release \ + --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ + --mount volume=etc-resolv,target=/etc/resolv.conf \ + --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ + --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ + --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ + --mount volume=lib-modules,target=/lib/modules \ + --volume run,kind=host,source=/run \ + --mount volume=run,target=/run \ + --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ + --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ + --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --mount volume=var-lib-calico,target=/var/lib/calico \ + --volume var-lib-docker,kind=host,source=/var/lib/docker \ + --mount volume=var-lib-docker,target=/var/lib/docker \ + --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ + --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ + --volume var-log,kind=host,source=/var/log \ + --mount volume=var-log,target=/var/log \ + --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ + --mount volume=opt-cni-bin,target=/opt/cni/bin \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ + --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -93,14 +111,6 @@ systemd: WantedBy=multi-user.target storage: files: - - path: /etc/kubernetes/kubelet.env - filesystem: root - mode: 0644 - contents: - inline: | - KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.17.0 - KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index b2baaa048..c48e90eb9 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -50,29 +50,46 @@ systemd: Description=Kubelet via Hyperkube Wants=rpc-statd.service [Service] - EnvironmentFile=/etc/kubernetes/kubelet.env - Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ - --volume=resolv,kind=host,source=/etc/resolv.conf \ - --mount volume=resolv,target=/etc/resolv.conf \ - --volume var-lib-cni,kind=host,source=/var/lib/cni \ - --mount volume=var-lib-cni,target=/var/lib/cni \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --hosts-entry=host \ - --insecure-options=image" ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/lib/coreos/kubelet-wrapper \ + ExecStart=/usr/bin/rkt run \ + --uuid-file-save=/var/cache/kubelet-pod.uuid \ + --stage1-from-dir=stage1-fly.aci \ + --hosts-entry host \ + --insecure-options=image \ + --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ + --mount volume=etc-kubernetes,target=/etc/kubernetes \ + --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ + --mount volume=etc-machine-id,target=/etc/machine-id \ + --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ + --mount volume=etc-os-release,target=/etc/os-release \ + --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ + --mount volume=etc-resolv,target=/etc/resolv.conf \ + --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ + --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ + --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ + --mount volume=lib-modules,target=/lib/modules \ + --volume run,kind=host,source=/run \ + --mount volume=run,target=/run \ + --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ + --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ + --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --mount volume=var-lib-calico,target=/var/lib/calico \ + --volume var-lib-docker,kind=host,source=/var/lib/docker \ + --mount volume=var-lib-docker,target=/var/lib/docker \ + --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ + --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ + --volume var-log,kind=host,source=/var/log \ + --mount volume=var-log,target=/var/log \ + --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ + --mount volume=opt-cni-bin,target=/opt/cni/bin \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ + --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -129,14 +146,6 @@ storage: contents: inline: | ${kubeconfig} - - path: /etc/kubernetes/kubelet.env - filesystem: root - mode: 0644 - contents: - inline: | - KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.17.0 - KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /opt/bootstrap/layout filesystem: root mode: 0544 diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml index 4d4bae02e..f6e5bd5d1 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml @@ -25,29 +25,46 @@ systemd: Description=Kubelet via Hyperkube Wants=rpc-statd.service [Service] - EnvironmentFile=/etc/kubernetes/kubelet.env - Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ - --volume=resolv,kind=host,source=/etc/resolv.conf \ - --mount volume=resolv,target=/etc/resolv.conf \ - --volume var-lib-cni,kind=host,source=/var/lib/cni \ - --mount volume=var-lib-cni,target=/var/lib/cni \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --hosts-entry=host \ - --insecure-options=image" ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/lib/coreos/kubelet-wrapper \ + ExecStart=/usr/bin/rkt run \ + --uuid-file-save=/var/cache/kubelet-pod.uuid \ + --stage1-from-dir=stage1-fly.aci \ + --hosts-entry host \ + --insecure-options=image \ + --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ + --mount volume=etc-kubernetes,target=/etc/kubernetes \ + --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ + --mount volume=etc-machine-id,target=/etc/machine-id \ + --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ + --mount volume=etc-os-release,target=/etc/os-release \ + --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ + --mount volume=etc-resolv,target=/etc/resolv.conf \ + --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ + --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ + --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ + --mount volume=lib-modules,target=/lib/modules \ + --volume run,kind=host,source=/run \ + --mount volume=run,target=/run \ + --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ + --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ + --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --mount volume=var-lib-calico,target=/var/lib/calico \ + --volume var-lib-docker,kind=host,source=/var/lib/docker \ + --mount volume=var-lib-docker,target=/var/lib/docker \ + --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ + --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ + --volume var-log,kind=host,source=/var/log \ + --mount volume=var-log,target=/var/log \ + --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ + --mount volume=opt-cni-bin,target=/opt/cni/bin \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ + --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -91,14 +108,6 @@ storage: contents: inline: | ${kubeconfig} - - path: /etc/kubernetes/kubelet.env - filesystem: root - mode: 0644 - contents: - inline: | - KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.17.0 - KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: