diff --git a/images/base/Dockerfile b/images/base/Dockerfile index d692c880af..804af0b669 100644 --- a/images/base/Dockerfile +++ b/images/base/Dockerfile @@ -27,19 +27,19 @@ ARG TARGETARCH # Configure containerd and runc binaries from kind-ci/containerd-nightlies repository # The repository contains latest stable releases and nightlies built for multiple architectures -ARG CONTAINERD_VERSION="1.6.2" +ARG CONTAINERD_VERSION="1.6.4" ARG CONTAINERD_BASE_URL="https://github.com/kind-ci/containerd-nightlies/releases/download/containerd-${CONTAINERD_VERSION}" ARG CONTAINERD_URL="${CONTAINERD_BASE_URL}/containerd-${CONTAINERD_VERSION}-linux-${TARGETARCH}.tar.gz" -ARG CONTAINERD_AMD64_SHA256SUM="be9efcdefdec84023ffba91cb55617b5ac3f178b6f3b1ccce314e3e2dad74d68" -ARG CONTAINERD_ARM64_SHA256SUM="ecc43b17bffe8fafa92ebc294859c3713d058872314fc1674349eff88709b02c" -ARG CONTAINERD_PPC64LE_SHA256SUM="6ea85f09e2a01f6f01666ac3842d6a210d905e1ed01e98beb8c4c7f547ef2e4d" -ARG CONTAINERD_S390X_SHA256SUM="cc1b634dc53a30bc06eb0bb8ff82e42ff13802e682f114cf24190055e2c95be0" +ARG CONTAINERD_AMD64_SHA256SUM="57d432c144871d3602b174232533619d7613b8453131a4fe30153ba9e2adae76" +ARG CONTAINERD_ARM64_SHA256SUM="8f566009eca685d4cd4ac351834a88f2c435bf299fe84dde9a975a989d806a2c" +ARG CONTAINERD_PPC64LE_SHA256SUM="39ba5b1974fb79559d20b1d644734c55505969df0ac67b0dc33a6499b15ba451" +ARG CONTAINERD_S390X_SHA256SUM="8b7f114ea9dfc6d3042cb31ec70ad7ae8afd3db3ef33ec6856c85b154c9be32e" ARG RUNC_URL="${CONTAINERD_BASE_URL}/runc.${TARGETARCH}" -ARG RUNC_AMD64_SHA256SUM="b99c4ed069047a7e90e3e23b1ad456a5b6d757ffebb2de1ad6c29ab66c050b82" -ARG RUNC_ARM64_SHA256SUM="c4de4327050cbe12bd53e6102426a4a0ed4ce343ac55537dc37ac044f9d7c33d" -ARG RUNC_PPC64LE_SHA256SUM="1d9bb1a547e954ebf173514b3215a4d07520eade43de3e677009116fadeea36c" -ARG RUNC_S390X_SHA256SUM="9084be4b463bcfb0908ee49678892afce6ca826bd8aca114c08bbf7041acf373" +ARG RUNC_AMD64_SHA256SUM="ebe475dc7b5e42f6c8940038582f8a642eab3e7da98693df83e5bd11ae598a1c" +ARG RUNC_ARM64_SHA256SUM="5f87f14a64563fe9c1822b661e049b7c0e6e038ac12082b6f91b76b751b6eca0" +ARG RUNC_PPC64LE_SHA256SUM="5ace7abfbca9a20e87567bf0718627f813b4430be398f6c01ca6fe2a0e04a364" +ARG RUNC_S390X_SHA256SUM="6e7d2c3d3e97df4fb8bcecd0e7b84ccbbba5b401a8c37da90cbd8f4714cf89ee" # Configure crictl binary from upstream ARG CRICTL_VERSION="v1.23.0" diff --git a/images/base/files/etc/containerd/config.toml b/images/base/files/etc/containerd/config.toml index a0baa684ac..13758e68c3 100644 --- a/images/base/files/etc/containerd/config.toml +++ b/images/base/files/etc/containerd/config.toml @@ -19,11 +19,18 @@ version = 2 runtime_type = "io.containerd.runc.v2" # Generated by "ctr oci spec" and modified at base container to mount poduct_uuid base_runtime_spec = "/etc/containerd/cri-base.json" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + # use systemd cgroup by default + SystemdCgroup = true # Setup a runtime with the magic name ("test-handler") used for Kubernetes # runtime class tests ... [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test-handler] + # same settings as runc runtime_type = "io.containerd.runc.v2" + base_runtime_spec = "/etc/containerd/cri-base.json" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test-handler.options] + SystemdCgroup = true [plugins."io.containerd.grpc.v1.cri"] # use fixed sandbox image diff --git a/images/base/files/etc/systemd/system/kubelet.service b/images/base/files/etc/systemd/system/kubelet.service index 4aa81b7874..4968e8f1c4 100644 --- a/images/base/files/etc/systemd/system/kubelet.service +++ b/images/base/files/etc/systemd/system/kubelet.service @@ -13,8 +13,11 @@ ConditionPathExists=/var/lib/kubelet/config.yaml ExecStart=/usr/bin/kubelet Restart=always StartLimitInterval=0 -# NOTE: kind deviates from upstream here with a lower RestartSecuse +# NOTE: kind deviates from upstream here with a lower RestartSec RestartSec=1s +# and here +CPUAccounting=true +MemoryAccounting=true [Install] WantedBy=multi-user.target diff --git a/images/base/files/usr/local/bin/create-kubelet-cgroup-v2 b/images/base/files/usr/local/bin/create-kubelet-cgroup-v2 index 120f60f3b5..8d0d0be8b9 100755 --- a/images/base/files/usr/local/bin/create-kubelet-cgroup-v2 +++ b/images/base/files/usr/local/bin/create-kubelet-cgroup-v2 @@ -44,3 +44,6 @@ ensure_subtree_control() { ensure_subtree_control / mkdir -p /sys/fs/cgroup/kubelet ensure_subtree_control /kubelet +# again for kubelet.slice for systemd cgroup driver +mkdir -p /sys/fs/cgroup/kubelet.slice +ensure_subtree_control /kubelet.slice diff --git a/images/base/files/usr/local/bin/entrypoint b/images/base/files/usr/local/bin/entrypoint index 7a8581ca1c..b665cd2f7e 100755 --- a/images/base/files/usr/local/bin/entrypoint +++ b/images/base/files/usr/local/bin/entrypoint @@ -294,10 +294,12 @@ fix_cgroup() { # "nesting" clusters, unless we instruct it to use a different cgroup root. # We do this, and when doing so we must fixup this alternative root # currently this is hardcoded to be /kubelet + # under systemd cgroup driver, kubelet appends .slice mount --make-rprivate /sys/fs/cgroup echo "${cgroup_subsystems}" | while IFS= read -r subsystem; do mount_kubelet_cgroup_root "/kubelet" "${subsystem}" + mount_kubelet_cgroup_root "/kubelet.slice" "${subsystem}" done } diff --git a/pkg/apis/config/defaults/image.go b/pkg/apis/config/defaults/image.go index 0e2c1600d6..60dd476656 100644 --- a/pkg/apis/config/defaults/image.go +++ b/pkg/apis/config/defaults/image.go @@ -18,4 +18,4 @@ limitations under the License. package defaults // Image is the default for the Config.Image field, aka the default node image. -const Image = "kindest/node:v1.23.6@sha256:b2921a38c34ac032ba2e53d734cdb33ef1e185b4a01c625d73a9f94f8e2d88e2" +const Image = "kindest/node:v1.24.0@sha256:406fd86d48eaf4c04c7280cd1d2ca1d61e7d0d61ddef0125cb097bc7b82ed6a1" diff --git a/pkg/build/nodeimage/buildcontext.go b/pkg/build/nodeimage/buildcontext.go index e8ad6f6b07..755adb4e19 100644 --- a/pkg/build/nodeimage/buildcontext.go +++ b/pkg/build/nodeimage/buildcontext.go @@ -27,7 +27,6 @@ import ( "sigs.k8s.io/kind/pkg/errors" "sigs.k8s.io/kind/pkg/exec" - "sigs.k8s.io/kind/pkg/fs" "sigs.k8s.io/kind/pkg/log" "sigs.k8s.io/kind/pkg/build/nodeimage/internal/container/docker" @@ -87,15 +86,10 @@ func (c *buildContext) buildImage(bits kube.Bits) error { return err } - c.logger.V(0).Info("Building in " + containerID) - - // helper we will use to run "build steps" - execInBuild := func(command string, args ...string) error { - return exec.InheritOutput(cmder.Command(command, args...)).Run() - } + c.logger.V(0).Info("Building in container: " + containerID) // make artifacts directory - if err = execInBuild("mkdir", "/kind/"); err != nil { + if err = cmder.Command("mkdir", "/kind/").Run(); err != nil { c.logger.Errorf("Image build Failed! Failed to make directory %v", err) return err } @@ -108,48 +102,45 @@ func (c *buildContext) buildImage(bits kube.Bits) error { if err := exec.Command("docker", "cp", binary, containerID+":"+nodePath).Run(); err != nil { return err } - if err := execInBuild("chmod", "+x", nodePath); err != nil { + if err := cmder.Command("chmod", "+x", nodePath).Run(); err != nil { return err } - if err := execInBuild("chown", "root:root", nodePath); err != nil { + if err := cmder.Command("chown", "root:root", nodePath).Run(); err != nil { return err } } // write version - // TODO: support grabbing version from a binary instead - if err := createFile(cmder, "/kind/version", bits.Version()); err != nil { - return err - } - - dir, err := fs.TempDir("", "kind-build") + // TODO: support grabbing version from a binary instead? + // This may or may not be a good idea ... + rawVersion := bits.Version() + parsedVersion, err := version.ParseSemantic(rawVersion) if err != nil { + return errors.Wrap(err, "invalid Kubernetes version") + } + if err := createFile(cmder, "/kind/version", rawVersion); err != nil { return err } - defer func() { - _ = os.RemoveAll(dir) - }() - // pre-pull images that were not part of the build - if _, err = c.prePullImages(bits, dir, containerID); err != nil { + // pre-pull images that were not part of the build and write CNI / storage + // manifests + if _, err = c.prePullImagesAndWriteManifests(bits, parsedVersion, containerID); err != nil { c.logger.Errorf("Image build Failed! Failed to pull Images: %v", err) return err } // Save the image changes to a new image - cmd := exec.Command( + if err = exec.Command( "docker", "commit", // we need to put this back after changing it when running the image "--change", `ENTRYPOINT [ "/usr/local/bin/entrypoint", "/sbin/init" ]`, containerID, c.image, - ) - exec.InheritOutput(cmd) - if err = cmd.Run(); err != nil { + ).Run(); err != nil { c.logger.Errorf("Image build Failed! Failed to save image: %v", err) return err } - c.logger.V(0).Info("Image build completed.") + c.logger.V(0).Infof("Image %q build completed.", c.image) return nil } @@ -167,7 +158,7 @@ func (c *buildContext) getBuiltImages(bits kube.Bits) (sets.String, error) { } // must be run after kubernetes has been installed on the node -func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([]string, error) { +func (c *buildContext) prePullImagesAndWriteManifests(bits kube.Bits, parsedVersion *version.Version, containerID string) ([]string, error) { // first get the images we actually built builtImages, err := c.getBuiltImages(bits) if err != nil { @@ -178,24 +169,6 @@ func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([ // helpers to run things in the build container cmder := docker.ContainerCmder(containerID) - // get the Kubernetes version we installed on the node - // we need this to ask kubeadm what images we need - rawVersion, err := exec.OutputLines(cmder.Command("cat", kubernetesVersionLocation)) - if err != nil { - c.logger.Errorf("Image build Failed! Failed to get Kubernetes version: %v", err) - return nil, err - } - if len(rawVersion) != 1 { - c.logger.Errorf("Image build Failed! Failed to get Kubernetes version: %v", err) - return nil, errors.New("invalid kubernetes version file") - } - - // parse version for comparison - ver, err := version.ParseSemantic(rawVersion[0]) - if err != nil { - return nil, err - } - // For kubernetes v1.15+ (actually 1.16 alpha versions) we may need to // drop the arch suffix from images to get the expected image archSuffix := "-" + c.arch @@ -224,18 +197,18 @@ func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([ // gets the list of images required by kubeadm requiredImages, err := exec.OutputLines(cmder.Command( - "kubeadm", "config", "images", "list", "--kubernetes-version", rawVersion[0], + "kubeadm", "config", "images", "list", "--kubernetes-version", bits.Version(), )) if err != nil { return nil, err } // replace pause image with our own - config, err := exec.Output(cmder.Command("cat", "/etc/containerd/config.toml")) + containerdConfig, err := exec.Output(cmder.Command("cat", containerdConfigPath)) if err != nil { return nil, err } - pauseImage, err := findSandboxImage(string(config)) + pauseImage, err := findSandboxImage(string(containerdConfig)) if err != nil { return nil, err } @@ -248,6 +221,12 @@ func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([ } requiredImages = append(requiredImages[:n], pauseImage) + if parsedVersion.LessThan(version.MustParseSemantic("v1.24.0")) { + if err := configureContainerdSystemdCgroupFalse(cmder, string(containerdConfig)); err != nil { + return nil, err + } + } + // write the default CNI manifest if err := createFile(cmder, defaultCNIManifestLocation, defaultCNIManifest); err != nil { c.logger.Errorf("Image build Failed! Failed write default CNI Manifest: %v", err) @@ -259,7 +238,7 @@ func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([ // write the default Storage manifest // in < 1.14 we need to use beta labels storageManifest := defaultStorageManifest - if ver.LessThan(version.MustParseSemantic("v1.14.0")) { + if parsedVersion.LessThan(version.MustParseSemantic("v1.14.0")) { storageManifest = strings.ReplaceAll(storageManifest, "kubernetes.io/os", "beta.kubernetes.io/os") } if err := createFile(cmder, defaultStorageManifestLocation, storageManifest); err != nil { @@ -269,13 +248,6 @@ func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([ // all builds should install the default storage driver images currently requiredImages = append(requiredImages, defaultStorageImages...) - // Create "images" subdir. - imagesDir := path.Join(dir, "bits", "images") - if err := os.MkdirAll(imagesDir, 0777); err != nil { - c.logger.Errorf("Image build Failed! Failed create local images dir: %v", err) - return nil, errors.Wrap(err, "failed to make images dir") - } - // setup image importer importer := newContainerdImporter(cmder) if err := importer.Prepare(); err != nil { @@ -371,7 +343,3 @@ func (c *buildContext) createBuildContainer() (id string, err error) { } return id, nil } - -func dockerBuildOsAndArch(arch string) string { - return "linux/" + arch -} diff --git a/pkg/build/nodeimage/containerd.go b/pkg/build/nodeimage/containerd.go new file mode 100644 index 0000000000..820af82c95 --- /dev/null +++ b/pkg/build/nodeimage/containerd.go @@ -0,0 +1,50 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nodeimage + +import ( + "strings" + + "sigs.k8s.io/kind/pkg/errors" + "sigs.k8s.io/kind/pkg/exec" + + "sigs.k8s.io/kind/pkg/internal/patch" +) + +const containerdConfigPath = "/etc/containerd/config.toml" + +const containerdConfigPatchSystemdCgroupFalse = ` +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = false + +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test-handler.options] + SystemdCgroup = false +` + +func configureContainerdSystemdCgroupFalse(containerCmdr exec.Cmder, config string) error { + patched, err := patch.TOML(config, []string{containerdConfigPatchSystemdCgroupFalse}, []string{}) + if err != nil { + return errors.Wrap(err, "failed to configure containerd SystemdCgroup=false") + } + err = containerCmdr.Command( + "cp", "/dev/stdin", containerdConfigPath, + ).SetStdin(strings.NewReader(patched)).Run() + if err != nil { + return errors.Wrap(err, "failed to configure containerd SystemdCgroup=false") + } + return nil +} diff --git a/pkg/build/nodeimage/defaults.go b/pkg/build/nodeimage/defaults.go index 7df3e693e8..d379086e51 100644 --- a/pkg/build/nodeimage/defaults.go +++ b/pkg/build/nodeimage/defaults.go @@ -20,4 +20,4 @@ package nodeimage const DefaultImage = "kindest/node:latest" // DefaultBaseImage is the default base image used -const DefaultBaseImage = "docker.io/kindest/base:v20220509-75bb7585" +const DefaultBaseImage = "docker.io/kindest/base:v20220509-9adca285" diff --git a/pkg/build/nodeimage/helpers.go b/pkg/build/nodeimage/helpers.go index 7887027d4d..4e7e3dbd71 100644 --- a/pkg/build/nodeimage/helpers.go +++ b/pkg/build/nodeimage/helpers.go @@ -49,3 +49,7 @@ func findSandboxImage(config string) (string, error) { } return match[1], nil } + +func dockerBuildOsAndArch(arch string) string { + return "linux/" + arch +} diff --git a/pkg/cluster/internal/create/actions/config/config.go b/pkg/cluster/internal/create/actions/config/config.go index d4c22ee1a2..1726198070 100644 --- a/pkg/cluster/internal/create/actions/config/config.go +++ b/pkg/cluster/internal/create/actions/config/config.go @@ -29,10 +29,10 @@ import ( "sigs.k8s.io/kind/pkg/cluster/internal/create/actions" "sigs.k8s.io/kind/pkg/cluster/internal/kubeadm" - "sigs.k8s.io/kind/pkg/cluster/internal/patch" "sigs.k8s.io/kind/pkg/cluster/internal/providers/common" "sigs.k8s.io/kind/pkg/cluster/nodeutils" "sigs.k8s.io/kind/pkg/internal/apis/config" + "sigs.k8s.io/kind/pkg/internal/patch" ) // Action implements action for creating the node config files diff --git a/pkg/cluster/internal/create/actions/installcni/cni.go b/pkg/cluster/internal/create/actions/installcni/cni.go index f7f6da0fc2..eeee6a2878 100644 --- a/pkg/cluster/internal/create/actions/installcni/cni.go +++ b/pkg/cluster/internal/create/actions/installcni/cni.go @@ -26,8 +26,8 @@ import ( "sigs.k8s.io/kind/pkg/internal/apis/config" "sigs.k8s.io/kind/pkg/cluster/internal/create/actions" - "sigs.k8s.io/kind/pkg/cluster/internal/patch" "sigs.k8s.io/kind/pkg/cluster/nodeutils" + "sigs.k8s.io/kind/pkg/internal/patch" ) type action struct{} diff --git a/pkg/cluster/internal/kubeadm/config.go b/pkg/cluster/internal/kubeadm/config.go index d66a2a92aa..2204f85ccf 100644 --- a/pkg/cluster/internal/kubeadm/config.go +++ b/pkg/cluster/internal/kubeadm/config.go @@ -102,10 +102,16 @@ type DerivedConfigData struct { KubeadmFeatureGates map[string]bool // IPv4 values take precedence over IPv6 by default, if true set IPv6 default values IPv6 bool + // kubelet cgroup driver, based on kubernetes version + CgroupDriver string } // Derive automatically derives DockerStableTag if not specified func (c *ConfigData) Derive() { + // default cgroup driver + // TODO: refactor and move all deriving logic to this method + c.CgroupDriver = "systemd" + // get the first address to use it as the API advertised address c.AdvertiseAddress = strings.Split(c.NodeAddress, ",")[0] @@ -248,7 +254,7 @@ metadata: # unblocks https://github.com/kubernetes/kubernetes/pull/99471 # TODO: consider switching to systemd instead # tracked in: https://github.com/kubernetes-sigs/kind/issues/1726 -cgroupDriver: cgroupfs +cgroupDriver: {{ .CgroupDriver }} # configure ipv6 addresses in IPv6 mode {{ if .IPv6 -}} address: "::" @@ -384,7 +390,7 @@ metadata: # unblocks https://github.com/kubernetes/kubernetes/pull/99471 # TODO: consider switching to systemd instead # tracked in: https://github.com/kubernetes-sigs/kind/issues/1726 -cgroupDriver: cgroupfs +cgroupDriver: {{ .CgroupDriver }} # configure ipv6 addresses in IPv6 mode {{ if .IPv6 -}} address: "::" @@ -525,7 +531,7 @@ metadata: # unblocks https://github.com/kubernetes/kubernetes/pull/99471 # TODO: consider switching to systemd instead # tracked in: https://github.com/kubernetes-sigs/kind/issues/1726 -cgroupDriver: cgroupfs +cgroupDriver: {{ .CgroupDriver }} # configure ipv6 addresses in IPv6 mode {{ if .IPv6 -}} address: "::" @@ -621,6 +627,13 @@ func Config(data ConfigData) (config string, err error) { data.KubeadmFeatureGates["IPv6DualStack"] = true } + // before 1.24 kind uses cgroupfs + // after 1.24 kind uses systemd starting in kind v0.13.0 + // before kind v0.13.0 kubernetes 1.24 wasn't released yet + if ver.LessThan(version.MustParseSemantic("v1.24.0")) { + data.CgroupDriver = "cgroupfs" + } + // execute the template var buff bytes.Buffer err = t.Execute(&buff, data) diff --git a/pkg/cluster/internal/patch/doc.go b/pkg/internal/patch/doc.go similarity index 100% rename from pkg/cluster/internal/patch/doc.go rename to pkg/internal/patch/doc.go diff --git a/pkg/cluster/internal/patch/json6902patch.go b/pkg/internal/patch/json6902patch.go similarity index 100% rename from pkg/cluster/internal/patch/json6902patch.go rename to pkg/internal/patch/json6902patch.go diff --git a/pkg/cluster/internal/patch/kubeyaml.go b/pkg/internal/patch/kubeyaml.go similarity index 100% rename from pkg/cluster/internal/patch/kubeyaml.go rename to pkg/internal/patch/kubeyaml.go diff --git a/pkg/cluster/internal/patch/kubeyaml_test.go b/pkg/internal/patch/kubeyaml_test.go similarity index 100% rename from pkg/cluster/internal/patch/kubeyaml_test.go rename to pkg/internal/patch/kubeyaml_test.go diff --git a/pkg/cluster/internal/patch/matchinfo.go b/pkg/internal/patch/matchinfo.go similarity index 100% rename from pkg/cluster/internal/patch/matchinfo.go rename to pkg/internal/patch/matchinfo.go diff --git a/pkg/cluster/internal/patch/mergepatch.go b/pkg/internal/patch/mergepatch.go similarity index 100% rename from pkg/cluster/internal/patch/mergepatch.go rename to pkg/internal/patch/mergepatch.go diff --git a/pkg/cluster/internal/patch/resource.go b/pkg/internal/patch/resource.go similarity index 100% rename from pkg/cluster/internal/patch/resource.go rename to pkg/internal/patch/resource.go diff --git a/pkg/cluster/internal/patch/toml.go b/pkg/internal/patch/toml.go similarity index 100% rename from pkg/cluster/internal/patch/toml.go rename to pkg/internal/patch/toml.go diff --git a/pkg/cluster/internal/patch/toml_test.go b/pkg/internal/patch/toml_test.go similarity index 100% rename from pkg/cluster/internal/patch/toml_test.go rename to pkg/internal/patch/toml_test.go