From b25641f1bbee9c0583c9cd1ec723f2bd312184e6 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 8 Nov 2020 11:46:39 -0800 Subject: [PATCH] Improve and tidy Fedora CoreOS etcd-member.service * Allow a snippet with a systemd dropin to set an alternate image via `ETCD_IMAGE`, for consistency across Fedora CoreOS and Flatcar Linux * Drop comments about integrating system containers with systemd-notify --- CHANGES.md | 2 + .../kubernetes/fcc/controller.yaml | 19 +++---- .../kubernetes/fcc/controller.yaml | 19 +++---- .../kubernetes/fcc/controller.yaml | 19 +++---- .../kubernetes/fcc/controller.yaml | 19 +++---- docs/advanced/customization.md | 56 +++++++++++++------ docs/topics/security.md | 2 +- .../kubernetes/fcc/controller.yaml | 19 +++---- 8 files changed, 76 insertions(+), 79 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 6bd3284eb..1977cb0df 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -10,6 +10,8 @@ Notable changes between versions. ### Fedora CoreOS +* Improve `etcd-member.service` systemd unit + * Allow a snippet with a systemd dropin to set an alternate image (e.g. mirror) * Fix local node delete oneshot on node shutdown ([#856](https://github.com/poseidon/typhoon/pull/855)) ### Flatcar Linux diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index b2c3ebe35..2ffef14a4 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -8,28 +8,25 @@ systemd: contents: | [Unit] Description=etcd (System Container) - Documentation=https://github.com/coreos/etcd + Documentation=https://github.com/etcd-io/etcd Wants=network-online.target network.target After=network-online.target [Service] - # https://github.com/opencontainers/runc/pull/1807 - # Type=notify - # NotifyAccess=exec + Environment=ETCD_IMAGE=quay.io/coreos/etcd:v3.4.12 Type=exec - Restart=on-failure - RestartSec=10s - TimeoutStartSec=0 - LimitNOFILE=40000 ExecStartPre=/bin/mkdir -p /var/lib/etcd ExecStartPre=-/usr/bin/podman rm etcd - #--volume $${NOTIFY_SOCKET}:/run/systemd/notify \ ExecStart=/usr/bin/podman run --name etcd \ --env-file /etc/etcd/etcd.env \ --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.4.12 + $${ETCD_IMAGE} ExecStop=/usr/bin/podman stop etcd + Restart=on-failure + RestartSec=10s + TimeoutStartSec=0 + LimitNOFILE=40000 [Install] WantedBy=multi-user.target - name: docker.service @@ -200,8 +197,6 @@ storage: mode: 0644 contents: inline: | - # TODO: Use a systemd dropin once podman v1.4.5 is avail. - NOTIFY_SOCKET=/run/systemd/notify ETCD_NAME=${etcd_name} ETCD_DATA_DIR=/var/lib/etcd ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379 diff --git a/azure/fedora-coreos/kubernetes/fcc/controller.yaml b/azure/fedora-coreos/kubernetes/fcc/controller.yaml index 740e9b961..1300e1d76 100644 --- a/azure/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/azure/fedora-coreos/kubernetes/fcc/controller.yaml @@ -8,28 +8,25 @@ systemd: contents: | [Unit] Description=etcd (System Container) - Documentation=https://github.com/coreos/etcd + Documentation=https://github.com/etcd-io/etcd Wants=network-online.target network.target After=network-online.target [Service] - # https://github.com/opencontainers/runc/pull/1807 - # Type=notify - # NotifyAccess=exec + Environment=ETCD_IMAGE=quay.io/coreos/etcd:v3.4.12 Type=exec - Restart=on-failure - RestartSec=10s - TimeoutStartSec=0 - LimitNOFILE=40000 ExecStartPre=/bin/mkdir -p /var/lib/etcd ExecStartPre=-/usr/bin/podman rm etcd - #--volume $${NOTIFY_SOCKET}:/run/systemd/notify \ ExecStart=/usr/bin/podman run --name etcd \ --env-file /etc/etcd/etcd.env \ --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.4.12 + $${ETCD_IMAGE} ExecStop=/usr/bin/podman stop etcd + Restart=on-failure + RestartSec=10s + TimeoutStartSec=0 + LimitNOFILE=40000 [Install] WantedBy=multi-user.target - name: docker.service @@ -199,8 +196,6 @@ storage: mode: 0644 contents: inline: | - # TODO: Use a systemd dropin once podman v1.4.5 is avail. - NOTIFY_SOCKET=/run/systemd/notify ETCD_NAME=${etcd_name} ETCD_DATA_DIR=/var/lib/etcd ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379 diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index e38220af6..5e79add46 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -8,28 +8,25 @@ systemd: contents: | [Unit] Description=etcd (System Container) - Documentation=https://github.com/coreos/etcd + Documentation=https://github.com/etcd-io/etcd Wants=network-online.target network.target After=network-online.target [Service] - # https://github.com/opencontainers/runc/pull/1807 - # Type=notify - # NotifyAccess=exec + Environment=ETCD_IMAGE=quay.io/coreos/etcd:v3.4.12 Type=exec - Restart=on-failure - RestartSec=10s - TimeoutStartSec=0 - LimitNOFILE=40000 ExecStartPre=/bin/mkdir -p /var/lib/etcd ExecStartPre=-/usr/bin/podman rm etcd - #--volume $${NOTIFY_SOCKET}:/run/systemd/notify \ ExecStart=/usr/bin/podman run --name etcd \ --env-file /etc/etcd/etcd.env \ --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.4.12 + $${ETCD_IMAGE} ExecStop=/usr/bin/podman stop etcd + Restart=on-failure + RestartSec=10s + TimeoutStartSec=0 + LimitNOFILE=40000 [Install] WantedBy=multi-user.target - name: docker.service @@ -210,8 +207,6 @@ storage: mode: 0644 contents: inline: | - # TODO: Use a systemd dropin once podman v1.4.5 is avail. - NOTIFY_SOCKET=/run/systemd/notify ETCD_NAME=${etcd_name} ETCD_DATA_DIR=/var/lib/etcd ETCD_ADVERTISE_CLIENT_URLS=https://${domain_name}:2379 diff --git a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml index 69326eb09..d4e201922 100644 --- a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml @@ -8,28 +8,25 @@ systemd: contents: | [Unit] Description=etcd (System Container) - Documentation=https://github.com/coreos/etcd + Documentation=https://github.com/etcd-io/etcd Wants=network-online.target network.target After=network-online.target [Service] - # https://github.com/opencontainers/runc/pull/1807 - # Type=notify - # NotifyAccess=exec + Environment=ETCD_IMAGE=quay.io/coreos/etcd:v3.4.12 Type=exec - Restart=on-failure - RestartSec=10s - TimeoutStartSec=0 - LimitNOFILE=40000 ExecStartPre=/bin/mkdir -p /var/lib/etcd ExecStartPre=-/usr/bin/podman rm etcd - #--volume $${NOTIFY_SOCKET}:/run/systemd/notify \ ExecStart=/usr/bin/podman run --name etcd \ --env-file /etc/etcd/etcd.env \ --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.4.12 + $${ETCD_IMAGE} ExecStop=/usr/bin/podman stop etcd + Restart=on-failure + RestartSec=10s + TimeoutStartSec=0 + LimitNOFILE=40000 [Install] WantedBy=multi-user.target - name: docker.service @@ -206,8 +203,6 @@ storage: mode: 0644 contents: inline: | - # TODO: Use a systemd dropin once podman v1.4.5 is avail. - NOTIFY_SOCKET=/run/systemd/notify ETCD_NAME=${etcd_name} ETCD_DATA_DIR=/var/lib/etcd ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379 diff --git a/docs/advanced/customization.md b/docs/advanced/customization.md index 4fd9f12fd..27ec0cf62 100644 --- a/docs/advanced/customization.md +++ b/docs/advanced/customization.md @@ -174,32 +174,52 @@ module "nemo" { To customize low-level Kubernetes control plane bootstrapping, see the [poseidon/terraform-render-bootstrap](https://github.com/poseidon/terraform-render-bootstrap) Terraform module. -## Kubelet +## System Images Typhoon publishes Kubelet [container images](/topics/security/#container-images) to Quay.io (default) and to Dockerhub (in case of a Quay [outage](https://github.com/poseidon/typhoon/issues/735) or breach). Quay automated builds also provide the option for fully verifiable tagged images (`build-{short_sha}`). -To set an alternative Kubelet image, use a snippet to set a systemd dropin. +To set an alternative etcd image or Kubelet image, use a snippet to set a systemd dropin. + +=== "Kubelet" + + ```yaml + # kubelet-image-override.yaml + variant: fcos <- remove for Flatcar Linux + version: 1.1.0 <- remove for Flatcar Linux + systemd: + units: + - name: kubelet.service + dropins: + - name: 10-image-override.conf + contents: | + [Service] + Environment=KUBELET_IMAGE=docker.io/psdn/kubelet:v1.18.3 + ``` + +=== "etcd" + + ```yaml + # etcd-image-override.yaml + variant: fcos <- remove for Flatcar Linux + version: 1.1.0 <- remove for Flatcar Linux + systemd: + units: + - name: etcd-member.service + dropins: + - name: 10-image-override.conf + contents: | + [Service] + Environment=ETCD_IMAGE=quay.io/mymirror/etcd:v3.4.12 + ``` + +Then reference the snippet in the cluster or worker pool definition. -``` -# host-image-override.yaml -variant: fcos <- remove for Flatcar Linux -version: 1.1.0 <- remove for Flatcar Linux -systemd: - units: - - name: kubelet.service - dropins: - - name: 10-image-override.conf - contents: | - [Service] - Environment=KUBELET_IMAGE=docker.io/psdn/kubelet:v1.18.3 -``` - -``` +```tf module "nemo" { ... worker_snippets = [ - file("./snippets/host-image-override.yaml") + file("./snippets/kubelet-image-override.yaml") ] ... } diff --git a/docs/topics/security.md b/docs/topics/security.md index 2c1400030..961b56b46 100644 --- a/docs/topics/security.md +++ b/docs/topics/security.md @@ -64,7 +64,7 @@ Two tag styles indicate the build strategy used. * Typhoon internal infra publishes single and multi-arch images (e.g. `v1.18.4`, `v1.18.4-amd64`, `v1.18.4-arm64`, `v1.18.4-2-g23228e6-amd64`, `v1.18.4-2-g23228e6-arm64`) * Quay and Dockerhub automated builds publish verifiable images (e.g. `build-SHA` on Quay, `build-TAG` on Dockerhub) -The Typhoon-built Kubelet image is used as the official image. Automated builds provide an alternative image for those preferring to trust images built by Quay/Dockerhub (albeit lacking multi-arch). To use the fallback registry or an alternative tag, see [customization](/advanced/customization/#kubelet). +The Typhoon-built Kubelet image is used as the official image. Automated builds provide an alternative image for those preferring to trust images built by Quay/Dockerhub (albeit lacking multi-arch). To use the fallback registry or an alternative tag, see [customization](/advanced/customization/#system-images). ### flannel-cni diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml index 358840901..04d03602d 100644 --- a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -8,28 +8,25 @@ systemd: contents: | [Unit] Description=etcd (System Container) - Documentation=https://github.com/coreos/etcd + Documentation=https://github.com/etcd-io/etcd Wants=network-online.target network.target After=network-online.target [Service] - # https://github.com/opencontainers/runc/pull/1807 - # Type=notify - # NotifyAccess=exec + Environment=ETCD_IMAGE=quay.io/coreos/etcd:v3.4.12 Type=exec - Restart=on-failure - RestartSec=10s - TimeoutStartSec=0 - LimitNOFILE=40000 ExecStartPre=/bin/mkdir -p /var/lib/etcd ExecStartPre=-/usr/bin/podman rm etcd - #--volume $${NOTIFY_SOCKET}:/run/systemd/notify \ ExecStart=/usr/bin/podman run --name etcd \ --env-file /etc/etcd/etcd.env \ --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.4.12 + $${ETCD_IMAGE} ExecStop=/usr/bin/podman stop etcd + Restart=on-failure + RestartSec=10s + TimeoutStartSec=0 + LimitNOFILE=40000 [Install] WantedBy=multi-user.target - name: docker.service @@ -199,8 +196,6 @@ storage: mode: 0644 contents: inline: | - # TODO: Use a systemd dropin once podman v1.4.5 is avail. - NOTIFY_SOCKET=/run/systemd/notify ETCD_NAME=${etcd_name} ETCD_DATA_DIR=/var/lib/etcd ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379