diff --git a/pkg/build/nodeimage/buildcontext.go b/pkg/build/nodeimage/buildcontext.go index 42650bde41..755adb4e19 100644 --- a/pkg/build/nodeimage/buildcontext.go +++ b/pkg/build/nodeimage/buildcontext.go @@ -113,13 +113,18 @@ func (c *buildContext) buildImage(bits kube.Bits) error { // write version // TODO: support grabbing version from a binary instead? // This may or may not be a good idea ... - if err := createFile(cmder, "/kind/version", bits.Version()); err != nil { + rawVersion := bits.Version() + parsedVersion, err := version.ParseSemantic(rawVersion) + if err != nil { + return errors.Wrap(err, "invalid Kubernetes version") + } + if err := createFile(cmder, "/kind/version", rawVersion); err != nil { return err } // pre-pull images that were not part of the build and write CNI / storage // manifests - if _, err = c.prePullImagesAndWriteManifests(bits, containerID); err != nil { + if _, err = c.prePullImagesAndWriteManifests(bits, parsedVersion, containerID); err != nil { c.logger.Errorf("Image build Failed! Failed to pull Images: %v", err) return err } @@ -153,7 +158,7 @@ func (c *buildContext) getBuiltImages(bits kube.Bits) (sets.String, error) { } // must be run after kubernetes has been installed on the node -func (c *buildContext) prePullImagesAndWriteManifests(bits kube.Bits, containerID string) ([]string, error) { +func (c *buildContext) prePullImagesAndWriteManifests(bits kube.Bits, parsedVersion *version.Version, containerID string) ([]string, error) { // first get the images we actually built builtImages, err := c.getBuiltImages(bits) if err != nil { @@ -164,13 +169,6 @@ func (c *buildContext) prePullImagesAndWriteManifests(bits kube.Bits, containerI // helpers to run things in the build container cmder := docker.ContainerCmder(containerID) - // parse version for comparison - rawVersion := bits.Version() - ver, err := version.ParseSemantic(rawVersion) - if err != nil { - return nil, err - } - // For kubernetes v1.15+ (actually 1.16 alpha versions) we may need to // drop the arch suffix from images to get the expected image archSuffix := "-" + c.arch @@ -199,18 +197,18 @@ func (c *buildContext) prePullImagesAndWriteManifests(bits kube.Bits, containerI // gets the list of images required by kubeadm requiredImages, err := exec.OutputLines(cmder.Command( - "kubeadm", "config", "images", "list", "--kubernetes-version", rawVersion, + "kubeadm", "config", "images", "list", "--kubernetes-version", bits.Version(), )) if err != nil { return nil, err } // replace pause image with our own - config, err := exec.Output(cmder.Command("cat", "/etc/containerd/config.toml")) + containerdConfig, err := exec.Output(cmder.Command("cat", containerdConfigPath)) if err != nil { return nil, err } - pauseImage, err := findSandboxImage(string(config)) + pauseImage, err := findSandboxImage(string(containerdConfig)) if err != nil { return nil, err } @@ -223,6 +221,12 @@ func (c *buildContext) prePullImagesAndWriteManifests(bits kube.Bits, containerI } requiredImages = append(requiredImages[:n], pauseImage) + if parsedVersion.LessThan(version.MustParseSemantic("v1.24.0")) { + if err := configureContainerdSystemdCgroupFalse(cmder, string(containerdConfig)); err != nil { + return nil, err + } + } + // write the default CNI manifest if err := createFile(cmder, defaultCNIManifestLocation, defaultCNIManifest); err != nil { c.logger.Errorf("Image build Failed! Failed write default CNI Manifest: %v", err) @@ -234,7 +238,7 @@ func (c *buildContext) prePullImagesAndWriteManifests(bits kube.Bits, containerI // write the default Storage manifest // in < 1.14 we need to use beta labels storageManifest := defaultStorageManifest - if ver.LessThan(version.MustParseSemantic("v1.14.0")) { + if parsedVersion.LessThan(version.MustParseSemantic("v1.14.0")) { storageManifest = strings.ReplaceAll(storageManifest, "kubernetes.io/os", "beta.kubernetes.io/os") } if err := createFile(cmder, defaultStorageManifestLocation, storageManifest); err != nil { diff --git a/pkg/build/nodeimage/containerd.go b/pkg/build/nodeimage/containerd.go new file mode 100644 index 0000000000..820af82c95 --- /dev/null +++ b/pkg/build/nodeimage/containerd.go @@ -0,0 +1,50 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nodeimage + +import ( + "strings" + + "sigs.k8s.io/kind/pkg/errors" + "sigs.k8s.io/kind/pkg/exec" + + "sigs.k8s.io/kind/pkg/internal/patch" +) + +const containerdConfigPath = "/etc/containerd/config.toml" + +const containerdConfigPatchSystemdCgroupFalse = ` +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = false + +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test-handler.options] + SystemdCgroup = false +` + +func configureContainerdSystemdCgroupFalse(containerCmdr exec.Cmder, config string) error { + patched, err := patch.TOML(config, []string{containerdConfigPatchSystemdCgroupFalse}, []string{}) + if err != nil { + return errors.Wrap(err, "failed to configure containerd SystemdCgroup=false") + } + err = containerCmdr.Command( + "cp", "/dev/stdin", containerdConfigPath, + ).SetStdin(strings.NewReader(patched)).Run() + if err != nil { + return errors.Wrap(err, "failed to configure containerd SystemdCgroup=false") + } + return nil +} diff --git a/pkg/cluster/internal/kubeadm/config.go b/pkg/cluster/internal/kubeadm/config.go index d66a2a92aa..2204f85ccf 100644 --- a/pkg/cluster/internal/kubeadm/config.go +++ b/pkg/cluster/internal/kubeadm/config.go @@ -102,10 +102,16 @@ type DerivedConfigData struct { KubeadmFeatureGates map[string]bool // IPv4 values take precedence over IPv6 by default, if true set IPv6 default values IPv6 bool + // kubelet cgroup driver, based on kubernetes version + CgroupDriver string } // Derive automatically derives DockerStableTag if not specified func (c *ConfigData) Derive() { + // default cgroup driver + // TODO: refactor and move all deriving logic to this method + c.CgroupDriver = "systemd" + // get the first address to use it as the API advertised address c.AdvertiseAddress = strings.Split(c.NodeAddress, ",")[0] @@ -248,7 +254,7 @@ metadata: # unblocks https://github.com/kubernetes/kubernetes/pull/99471 # TODO: consider switching to systemd instead # tracked in: https://github.com/kubernetes-sigs/kind/issues/1726 -cgroupDriver: cgroupfs +cgroupDriver: {{ .CgroupDriver }} # configure ipv6 addresses in IPv6 mode {{ if .IPv6 -}} address: "::" @@ -384,7 +390,7 @@ metadata: # unblocks https://github.com/kubernetes/kubernetes/pull/99471 # TODO: consider switching to systemd instead # tracked in: https://github.com/kubernetes-sigs/kind/issues/1726 -cgroupDriver: cgroupfs +cgroupDriver: {{ .CgroupDriver }} # configure ipv6 addresses in IPv6 mode {{ if .IPv6 -}} address: "::" @@ -525,7 +531,7 @@ metadata: # unblocks https://github.com/kubernetes/kubernetes/pull/99471 # TODO: consider switching to systemd instead # tracked in: https://github.com/kubernetes-sigs/kind/issues/1726 -cgroupDriver: cgroupfs +cgroupDriver: {{ .CgroupDriver }} # configure ipv6 addresses in IPv6 mode {{ if .IPv6 -}} address: "::" @@ -621,6 +627,13 @@ func Config(data ConfigData) (config string, err error) { data.KubeadmFeatureGates["IPv6DualStack"] = true } + // before 1.24 kind uses cgroupfs + // after 1.24 kind uses systemd starting in kind v0.13.0 + // before kind v0.13.0 kubernetes 1.24 wasn't released yet + if ver.LessThan(version.MustParseSemantic("v1.24.0")) { + data.CgroupDriver = "cgroupfs" + } + // execute the template var buff bytes.Buffer err = t.Execute(&buff, data)