Merge pull request #2737 from BenTheElder/systemd-driver

Use systemd cgroup driver for v1.24.0+

images/base/Dockerfile

@@ -27,19 +27,19 @@ ARG TARGETARCH
 
 # Configure containerd and runc binaries from kind-ci/containerd-nightlies repository
 # The repository contains latest stable releases and nightlies built for multiple architectures
-ARG CONTAINERD_VERSION="1.6.2"
+ARG CONTAINERD_VERSION="1.6.4"
 ARG CONTAINERD_BASE_URL="https://github.com/kind-ci/containerd-nightlies/releases/download/containerd-${CONTAINERD_VERSION}"
 ARG CONTAINERD_URL="${CONTAINERD_BASE_URL}/containerd-${CONTAINERD_VERSION}-linux-${TARGETARCH}.tar.gz"
-ARG CONTAINERD_AMD64_SHA256SUM="be9efcdefdec84023ffba91cb55617b5ac3f178b6f3b1ccce314e3e2dad74d68"
-ARG CONTAINERD_ARM64_SHA256SUM="ecc43b17bffe8fafa92ebc294859c3713d058872314fc1674349eff88709b02c"
-ARG CONTAINERD_PPC64LE_SHA256SUM="6ea85f09e2a01f6f01666ac3842d6a210d905e1ed01e98beb8c4c7f547ef2e4d"
-ARG CONTAINERD_S390X_SHA256SUM="cc1b634dc53a30bc06eb0bb8ff82e42ff13802e682f114cf24190055e2c95be0"
+ARG CONTAINERD_AMD64_SHA256SUM="57d432c144871d3602b174232533619d7613b8453131a4fe30153ba9e2adae76"
+ARG CONTAINERD_ARM64_SHA256SUM="8f566009eca685d4cd4ac351834a88f2c435bf299fe84dde9a975a989d806a2c"
+ARG CONTAINERD_PPC64LE_SHA256SUM="39ba5b1974fb79559d20b1d644734c55505969df0ac67b0dc33a6499b15ba451"
+ARG CONTAINERD_S390X_SHA256SUM="8b7f114ea9dfc6d3042cb31ec70ad7ae8afd3db3ef33ec6856c85b154c9be32e"
 
 ARG RUNC_URL="${CONTAINERD_BASE_URL}/runc.${TARGETARCH}"
-ARG RUNC_AMD64_SHA256SUM="b99c4ed069047a7e90e3e23b1ad456a5b6d757ffebb2de1ad6c29ab66c050b82"
-ARG RUNC_ARM64_SHA256SUM="c4de4327050cbe12bd53e6102426a4a0ed4ce343ac55537dc37ac044f9d7c33d"
-ARG RUNC_PPC64LE_SHA256SUM="1d9bb1a547e954ebf173514b3215a4d07520eade43de3e677009116fadeea36c"
-ARG RUNC_S390X_SHA256SUM="9084be4b463bcfb0908ee49678892afce6ca826bd8aca114c08bbf7041acf373"
+ARG RUNC_AMD64_SHA256SUM="ebe475dc7b5e42f6c8940038582f8a642eab3e7da98693df83e5bd11ae598a1c"
+ARG RUNC_ARM64_SHA256SUM="5f87f14a64563fe9c1822b661e049b7c0e6e038ac12082b6f91b76b751b6eca0"
+ARG RUNC_PPC64LE_SHA256SUM="5ace7abfbca9a20e87567bf0718627f813b4430be398f6c01ca6fe2a0e04a364"
+ARG RUNC_S390X_SHA256SUM="6e7d2c3d3e97df4fb8bcecd0e7b84ccbbba5b401a8c37da90cbd8f4714cf89ee"
 
 # Configure crictl binary from upstream
 ARG CRICTL_VERSION="v1.23.0"

images/base/files/etc/containerd/config.toml

@@ -19,11 +19,18 @@ version = 2
   runtime_type = "io.containerd.runc.v2"
   # Generated by "ctr oci spec" and modified at base container to mount poduct_uuid
   base_runtime_spec = "/etc/containerd/cri-base.json"
+  [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+    # use systemd cgroup by default
+    SystemdCgroup = true
 
 # Setup a runtime with the magic name ("test-handler") used for Kubernetes
 # runtime class tests ...
 [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test-handler]
+  # same settings as runc
   runtime_type = "io.containerd.runc.v2"
+  base_runtime_spec = "/etc/containerd/cri-base.json"
+  [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test-handler.options]
+    SystemdCgroup = true
 
 [plugins."io.containerd.grpc.v1.cri"]
   # use fixed sandbox image

images/base/files/etc/systemd/system/kubelet.service

@@ -13,8 +13,11 @@ ConditionPathExists=/var/lib/kubelet/config.yaml
 ExecStart=/usr/bin/kubelet
 Restart=always
 StartLimitInterval=0
-# NOTE: kind deviates from upstream here with a lower RestartSecuse
+# NOTE: kind deviates from upstream here with a lower RestartSec
 RestartSec=1s
+# and here
+CPUAccounting=true
+MemoryAccounting=true
 
 [Install]
 WantedBy=multi-user.target

images/base/files/usr/local/bin/create-kubelet-cgroup-v2

@@ -44,3 +44,6 @@ ensure_subtree_control() {
 ensure_subtree_control /
 mkdir -p /sys/fs/cgroup/kubelet
 ensure_subtree_control /kubelet
+# again for kubelet.slice for systemd cgroup driver
+mkdir -p /sys/fs/cgroup/kubelet.slice
+ensure_subtree_control /kubelet.slice

images/base/files/usr/local/bin/entrypoint

@@ -294,10 +294,12 @@ fix_cgroup() {
   # "nesting" clusters, unless we instruct it to use a different cgroup root.
   # We do this, and when doing so we must fixup this alternative root
   # currently this is hardcoded to be /kubelet
+  # under systemd cgroup driver, kubelet appends .slice
   mount --make-rprivate /sys/fs/cgroup
   echo "${cgroup_subsystems}" |
   while IFS= read -r subsystem; do
     mount_kubelet_cgroup_root "/kubelet" "${subsystem}"
+    mount_kubelet_cgroup_root "/kubelet.slice" "${subsystem}"
   done
 }
 

pkg/apis/config/defaults/image.go

@@ -18,4 +18,4 @@ limitations under the License.
 package defaults
 
 // Image is the default for the Config.Image field, aka the default node image.
-const Image = "kindest/node:v1.23.6@sha256:b2921a38c34ac032ba2e53d734cdb33ef1e185b4a01c625d73a9f94f8e2d88e2"
+const Image = "kindest/node:v1.24.0@sha256:406fd86d48eaf4c04c7280cd1d2ca1d61e7d0d61ddef0125cb097bc7b82ed6a1"

pkg/build/nodeimage/buildcontext.go

@@ -27,7 +27,6 @@ import (
 
 	"sigs.k8s.io/kind/pkg/errors"
 	"sigs.k8s.io/kind/pkg/exec"
-	"sigs.k8s.io/kind/pkg/fs"
 	"sigs.k8s.io/kind/pkg/log"
 
 	"sigs.k8s.io/kind/pkg/build/nodeimage/internal/container/docker"
@@ -87,15 +86,10 @@ func (c *buildContext) buildImage(bits kube.Bits) error {
 		return err
 	}
 
-	c.logger.V(0).Info("Building in " + containerID)
-
-	// helper we will use to run "build steps"
-	execInBuild := func(command string, args ...string) error {
-		return exec.InheritOutput(cmder.Command(command, args...)).Run()
-	}
+	c.logger.V(0).Info("Building in container: " + containerID)
 
 	// make artifacts directory
-	if err = execInBuild("mkdir", "/kind/"); err != nil {
+	if err = cmder.Command("mkdir", "/kind/").Run(); err != nil {
 		c.logger.Errorf("Image build Failed! Failed to make directory %v", err)
 		return err
 	}
@@ -108,48 +102,45 @@ func (c *buildContext) buildImage(bits kube.Bits) error {
 		if err := exec.Command("docker", "cp", binary, containerID+":"+nodePath).Run(); err != nil {
 			return err
 		}
-		if err := execInBuild("chmod", "+x", nodePath); err != nil {
+		if err := cmder.Command("chmod", "+x", nodePath).Run(); err != nil {
 			return err
 		}
-		if err := execInBuild("chown", "root:root", nodePath); err != nil {
+		if err := cmder.Command("chown", "root:root", nodePath).Run(); err != nil {
 			return err
 		}
 	}
 
 	// write version
-	// TODO: support grabbing version from a binary instead
-	if err := createFile(cmder, "/kind/version", bits.Version()); err != nil {
-		return err
-	}
-
-	dir, err := fs.TempDir("", "kind-build")
+	// TODO: support grabbing version from a binary instead?
+	// This may or may not be a good idea ...
+	rawVersion := bits.Version()
+	parsedVersion, err := version.ParseSemantic(rawVersion)
 	if err != nil {
+		return errors.Wrap(err, "invalid Kubernetes version")
+	}
+	if err := createFile(cmder, "/kind/version", rawVersion); err != nil {
 		return err
 	}
-	defer func() {
-		_ = os.RemoveAll(dir)
-	}()
 
-	// pre-pull images that were not part of the build
-	if _, err = c.prePullImages(bits, dir, containerID); err != nil {
+	// pre-pull images that were not part of the build and write CNI / storage
+	// manifests
+	if _, err = c.prePullImagesAndWriteManifests(bits, parsedVersion, containerID); err != nil {
 		c.logger.Errorf("Image build Failed! Failed to pull Images: %v", err)
 		return err
 	}
 
 	// Save the image changes to a new image
-	cmd := exec.Command(
+	if err = exec.Command(
 		"docker", "commit",
 		// we need to put this back after changing it when running the image
 		"--change", `ENTRYPOINT [ "/usr/local/bin/entrypoint", "/sbin/init" ]`,
 		containerID, c.image,
-	)
-	exec.InheritOutput(cmd)
-	if err = cmd.Run(); err != nil {
+	).Run(); err != nil {
 		c.logger.Errorf("Image build Failed! Failed to save image: %v", err)
 		return err
 	}
 
-	c.logger.V(0).Info("Image build completed.")
+	c.logger.V(0).Infof("Image %q build completed.", c.image)
 	return nil
 }
 
@@ -167,7 +158,7 @@ func (c *buildContext) getBuiltImages(bits kube.Bits) (sets.String, error) {
 }
 
 // must be run after kubernetes has been installed on the node
-func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([]string, error) {
+func (c *buildContext) prePullImagesAndWriteManifests(bits kube.Bits, parsedVersion *version.Version, containerID string) ([]string, error) {
 	// first get the images we actually built
 	builtImages, err := c.getBuiltImages(bits)
 	if err != nil {
@@ -178,24 +169,6 @@ func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([
 	// helpers to run things in the build container
 	cmder := docker.ContainerCmder(containerID)
 
-	// get the Kubernetes version we installed on the node
-	// we need this to ask kubeadm what images we need
-	rawVersion, err := exec.OutputLines(cmder.Command("cat", kubernetesVersionLocation))
-	if err != nil {
-		c.logger.Errorf("Image build Failed! Failed to get Kubernetes version: %v", err)
-		return nil, err
-	}
-	if len(rawVersion) != 1 {
-		c.logger.Errorf("Image build Failed! Failed to get Kubernetes version: %v", err)
-		return nil, errors.New("invalid kubernetes version file")
-	}
-
-	// parse version for comparison
-	ver, err := version.ParseSemantic(rawVersion[0])
-	if err != nil {
-		return nil, err
-	}
-
 	// For kubernetes v1.15+ (actually 1.16 alpha versions) we may need to
 	// drop the arch suffix from images to get the expected image
 	archSuffix := "-" + c.arch
@@ -224,18 +197,18 @@ func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([
 
 	// gets the list of images required by kubeadm
 	requiredImages, err := exec.OutputLines(cmder.Command(
-		"kubeadm", "config", "images", "list", "--kubernetes-version", rawVersion[0],
+		"kubeadm", "config", "images", "list", "--kubernetes-version", bits.Version(),
 	))
 	if err != nil {
 		return nil, err
 	}
 
 	// replace pause image with our own
-	config, err := exec.Output(cmder.Command("cat", "/etc/containerd/config.toml"))
+	containerdConfig, err := exec.Output(cmder.Command("cat", containerdConfigPath))
 	if err != nil {
 		return nil, err
 	}
-	pauseImage, err := findSandboxImage(string(config))
+	pauseImage, err := findSandboxImage(string(containerdConfig))
 	if err != nil {
 		return nil, err
 	}
@@ -248,6 +221,12 @@ func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([
 	}
 	requiredImages = append(requiredImages[:n], pauseImage)
 
+	if parsedVersion.LessThan(version.MustParseSemantic("v1.24.0")) {
+		if err := configureContainerdSystemdCgroupFalse(cmder, string(containerdConfig)); err != nil {
+			return nil, err
+		}
+	}
+
 	// write the default CNI manifest
 	if err := createFile(cmder, defaultCNIManifestLocation, defaultCNIManifest); err != nil {
 		c.logger.Errorf("Image build Failed! Failed write default CNI Manifest: %v", err)
@@ -259,7 +238,7 @@ func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([
 	// write the default Storage manifest
 	// in < 1.14 we need to use beta labels
 	storageManifest := defaultStorageManifest
-	if ver.LessThan(version.MustParseSemantic("v1.14.0")) {
+	if parsedVersion.LessThan(version.MustParseSemantic("v1.14.0")) {
 		storageManifest = strings.ReplaceAll(storageManifest, "kubernetes.io/os", "beta.kubernetes.io/os")
 	}
 	if err := createFile(cmder, defaultStorageManifestLocation, storageManifest); err != nil {
@@ -269,13 +248,6 @@ func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([
 	// all builds should install the default storage driver images currently
 	requiredImages = append(requiredImages, defaultStorageImages...)
 
-	// Create "images" subdir.
-	imagesDir := path.Join(dir, "bits", "images")
-	if err := os.MkdirAll(imagesDir, 0777); err != nil {
-		c.logger.Errorf("Image build Failed! Failed create local images dir: %v", err)
-		return nil, errors.Wrap(err, "failed to make images dir")
-	}
-
 	// setup image importer
 	importer := newContainerdImporter(cmder)
 	if err := importer.Prepare(); err != nil {
@@ -371,7 +343,3 @@ func (c *buildContext) createBuildContainer() (id string, err error) {
 	}
 	return id, nil
 }
-
-func dockerBuildOsAndArch(arch string) string {
-	return "linux/" + arch
-}

pkg/build/nodeimage/containerd.go

@@ -0,0 +1,50 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nodeimage
+
+import (
+	"strings"
+
+	"sigs.k8s.io/kind/pkg/errors"
+	"sigs.k8s.io/kind/pkg/exec"
+
+	"sigs.k8s.io/kind/pkg/internal/patch"
+)
+
+const containerdConfigPath = "/etc/containerd/config.toml"
+
+const containerdConfigPatchSystemdCgroupFalse = `
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+  SystemdCgroup = false
+
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test-handler.options]
+  SystemdCgroup = false
+`
+
+func configureContainerdSystemdCgroupFalse(containerCmdr exec.Cmder, config string) error {
+	patched, err := patch.TOML(config, []string{containerdConfigPatchSystemdCgroupFalse}, []string{})
+	if err != nil {
+		return errors.Wrap(err, "failed to configure containerd SystemdCgroup=false")
+	}
+	err = containerCmdr.Command(
+		"cp", "/dev/stdin", containerdConfigPath,
+	).SetStdin(strings.NewReader(patched)).Run()
+	if err != nil {
+		return errors.Wrap(err, "failed to configure containerd SystemdCgroup=false")
+	}
+	return nil
+}

pkg/build/nodeimage/defaults.go

@@ -20,4 +20,4 @@ package nodeimage
 const DefaultImage = "kindest/node:latest"
 
 // DefaultBaseImage is the default base image used
-const DefaultBaseImage = "docker.io/kindest/base:v20220509-75bb7585"
+const DefaultBaseImage = "docker.io/kindest/base:v20220509-9adca285"

pkg/build/nodeimage/helpers.go

@@ -49,3 +49,7 @@ func findSandboxImage(config string) (string, error) {
 	}
 	return match[1], nil
 }
+
+func dockerBuildOsAndArch(arch string) string {
+	return "linux/" + arch
+}

pkg/cluster/internal/create/actions/config/config.go

@@ -29,10 +29,10 @@ import (
 
 	"sigs.k8s.io/kind/pkg/cluster/internal/create/actions"
 	"sigs.k8s.io/kind/pkg/cluster/internal/kubeadm"
-	"sigs.k8s.io/kind/pkg/cluster/internal/patch"
 	"sigs.k8s.io/kind/pkg/cluster/internal/providers/common"
 	"sigs.k8s.io/kind/pkg/cluster/nodeutils"
 	"sigs.k8s.io/kind/pkg/internal/apis/config"
+	"sigs.k8s.io/kind/pkg/internal/patch"
 )
 
 // Action implements action for creating the node config files

pkg/cluster/internal/create/actions/installcni/cni.go

@@ -26,8 +26,8 @@ import (
 	"sigs.k8s.io/kind/pkg/internal/apis/config"
 
 	"sigs.k8s.io/kind/pkg/cluster/internal/create/actions"
-	"sigs.k8s.io/kind/pkg/cluster/internal/patch"
 	"sigs.k8s.io/kind/pkg/cluster/nodeutils"
+	"sigs.k8s.io/kind/pkg/internal/patch"
 )
 
 type action struct{}