Skip to content
This repository has been archived by the owner on Apr 11, 2024. It is now read-only.

Commit

Permalink
feat: Make containerd restart its own patch
Browse files Browse the repository at this point in the history
  • Loading branch information
dlipovetsky committed Mar 30, 2024
1 parent f04cd61 commit 3232a20
Show file tree
Hide file tree
Showing 10 changed files with 200 additions and 37 deletions.
6 changes: 6 additions & 0 deletions pkg/handlers/aws/mutation/metapatch_handler_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
regiontests "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/aws/mutation/region/tests"
"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/clusterconfig"
auditpolicytests "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/auditpolicy/tests"
containerdrestarttests "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/containerdrestart/tests"
"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/etcd"
etcdtests "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/etcd/tests"
"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/extraapiservercertsans"
Expand Down Expand Up @@ -188,4 +189,9 @@ func TestGeneratePatches(t *testing.T) {
v1alpha1.AWSVariableName,
controlplaneloadbalancer.VariableName,
)

containerdrestarttests.TestGeneratePatches(
t,
metaPatchGeneratorFunc(mgr),
)
}
6 changes: 6 additions & 0 deletions pkg/handlers/docker/mutation/metapatch_handler_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
dockerworkerconfig "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/docker/workerconfig"
"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/clusterconfig"
auditpolicytests "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/auditpolicy/tests"
containerdrestarttests "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/containerdrestart/tests"
"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/etcd"
etcdtests "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/etcd/tests"
"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/extraapiservercertsans"
Expand Down Expand Up @@ -112,4 +113,9 @@ func TestGeneratePatches(t *testing.T) {
clusterconfig.MetaVariableName,
mirrors.GlobalMirrorVariableName,
)

containerdrestarttests.TestGeneratePatches(
t,
metaPatchGeneratorFunc(mgr),
)
}
86 changes: 86 additions & 0 deletions pkg/handlers/generic/mutation/containerdrestart/inject.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
// Copyright 2023 D2iQ, Inc. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package containerdrestart

import (
"context"

apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
ctrl "sigs.k8s.io/controller-runtime"
ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"

"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches"
"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches/selectors"
)

type globalMirrorPatchHandler struct{}

func NewPatch() *globalMirrorPatchHandler {
return &globalMirrorPatchHandler{}
}

func (h *globalMirrorPatchHandler) Mutate(
ctx context.Context,
obj *unstructured.Unstructured,
vars map[string]apiextensionsv1.JSON,
holderRef runtimehooksv1.HolderReference,
clusterKey ctrlclient.ObjectKey,
) error {
log := ctrl.LoggerFrom(ctx).WithValues(
"holderRef", holderRef,
)

file, command := generateContainerdRestartScript()

if err := patches.MutateIfApplicable(
obj, vars, &holderRef, selectors.ControlPlane(), log,
func(obj *controlplanev1.KubeadmControlPlaneTemplate) error {
log.WithValues(
"patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(),
"patchedObjectName", ctrlclient.ObjectKeyFromObject(obj),
).Info("adding containerd restart script to control plane kubeadm config spec")
obj.Spec.Template.Spec.KubeadmConfigSpec.Files = append(
obj.Spec.Template.Spec.KubeadmConfigSpec.Files,
file,
)

log.WithValues(
"patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(),
"patchedObjectName", ctrlclient.ObjectKeyFromObject(obj),
).Info("adding containerd restart command to control plane kubeadm config spec")
obj.Spec.Template.Spec.KubeadmConfigSpec.PreKubeadmCommands = append(
obj.Spec.Template.Spec.KubeadmConfigSpec.PreKubeadmCommands,
command,
)

return nil
}); err != nil {
return err
}

if err := patches.MutateIfApplicable(
obj, vars, &holderRef, selectors.WorkersKubeadmConfigTemplateSelector(), log,
func(obj *bootstrapv1.KubeadmConfigTemplate) error {
log.WithValues(
"patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(),
"patchedObjectName", ctrlclient.ObjectKeyFromObject(obj),
).Info("adding containerd restart script to worker node kubeadm config template")
obj.Spec.Template.Spec.Files = append(obj.Spec.Template.Spec.Files, file)

log.WithValues(
"patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(),
"patchedObjectName", ctrlclient.ObjectKeyFromObject(obj),
).Info("adding containerd restart command to worker node kubeadm config template")
obj.Spec.Template.Spec.PreKubeadmCommands = append(obj.Spec.Template.Spec.PreKubeadmCommands, command)

return nil
}); err != nil {
return err
}

return nil
}
25 changes: 25 additions & 0 deletions pkg/handlers/generic/mutation/containerdrestart/restart.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
package containerdrestart

import (
_ "embed"

bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
)

const (
ContainerdRestartScriptOnRemote = "/etc/containerd/restart.sh"
ContainerdRestartScriptOnRemoteCommand = "/bin/bash " + ContainerdRestartScriptOnRemote
)

//go:embed templates/containerd-restart.sh
var containerdRestartScript []byte

//nolint:gocritic // no need for named return values
func generateContainerdRestartScript() (bootstrapv1.File, string) {
return bootstrapv1.File{
Path: ContainerdRestartScriptOnRemote,
Content: string(containerdRestartScript),
Permissions: "0700",
},
ContainerdRestartScriptOnRemoteCommand
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
// Copyright 2023 D2iQ, Inc. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

package tests

import (
"testing"

"github.com/onsi/gomega"

"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation"
"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest"
"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest/request"
"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/containerdrestart"
)

func TestGeneratePatches(
t *testing.T,
generatorFunc func() mutation.GeneratePatches,
) {
t.Helper()

capitest.ValidateGeneratePatches(
t,
generatorFunc,
capitest.PatchTestDef{
Name: "restart script and command added to control plane kubeadm config spec",
RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""),
ExpectedPatchMatchers: []capitest.JSONPatchMatcher{
{
Operation: "add",
Path: "/spec/template/spec/kubeadmConfigSpec/files",
ValueMatcher: gomega.ContainElements(
gomega.HaveKeyWithValue(
"path", containerdrestart.ContainerdRestartScriptOnRemote,
),
),
},
{
Operation: "add",
Path: "/spec/template/spec/kubeadmConfigSpec/preKubeadmCommands",
ValueMatcher: gomega.ContainElements(
containerdrestart.ContainerdRestartScriptOnRemoteCommand,
),
},
},
},
capitest.PatchTestDef{
Name: "restart script and command added to worker node kubeadm config template",
RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""),
ExpectedPatchMatchers: []capitest.JSONPatchMatcher{
{
Operation: "add",
Path: "/spec/template/spec/kubeadmConfigSpec/files",
ValueMatcher: gomega.ContainElements(
gomega.HaveKeyWithValue(
"path", containerdrestart.ContainerdRestartScriptOnRemote,
),
),
},
{
Operation: "add",
Path: "/spec/template/spec/kubeadmConfigSpec/preKubeadmCommands",
ValueMatcher: gomega.ContainElements(
containerdrestart.ContainerdRestartScriptOnRemoteCommand,
),
},
},
},
)
}
6 changes: 6 additions & 0 deletions pkg/handlers/generic/mutation/handlers.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation"
"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/aws/mutation/cni/calico"
"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/auditpolicy"
"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/containerdrestart"
"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/etcd"
"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/extraapiservercertsans"
"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/httpproxy"
Expand All @@ -28,5 +29,10 @@ func MetaMutators(mgr manager.Manager) []mutation.MetaMutator {
credentials.NewPatch(mgr.GetClient()),
mirrors.NewPatch(mgr.GetClient()),
calico.NewPatch(),

// Some patches may have changed containerd configuration.
// We must restart containerd for the configuration to take effect.
// Therefore, we must apply this patch last.
containerdrestart.NewPatch(),
}
}
4 changes: 0 additions & 4 deletions pkg/handlers/generic/mutation/mirrors/inject.go
Original file line number Diff line number Diff line change
Expand Up @@ -191,10 +191,6 @@ func generateFilesAndCommands(
}
files = append(files, applyPatchesFile...)
commands = append(commands, applyPatchesCommand)
// generate Containerd restart script and command
restartFile, restartCommand := generateContainerdRestartScript()
files = append(files, restartFile...)
commands = append(commands, restartCommand)

return files, commands, err
}
17 changes: 0 additions & 17 deletions pkg/handlers/generic/mutation/mirrors/mirror.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,6 @@ const (
containerdPatchesDirOnRemote = "/etc/containerd/cre.d"
containerdApplyPatchesScriptOnRemote = "/etc/containerd/apply-patches.sh"
containerdApplyPatchesScriptOnRemoteCommand = "/bin/bash " + containerdApplyPatchesScriptOnRemote

containerdRestartScriptOnRemote = "/etc/containerd/restart.sh"
containerdRestartScriptOnRemoteCommand = "/bin/bash " + containerdRestartScriptOnRemote
)

var (
Expand All @@ -50,9 +47,6 @@ var (

//go:embed templates/containerd-apply-patches.sh.gotmpl
containerdApplyConfigPatchesScript []byte

//go:embed templates/containerd-restart.sh
containerdRestartScript []byte
)

type mirrorConfig struct {
Expand Down Expand Up @@ -226,14 +220,3 @@ func generateContainerdApplyPatchesScript() ([]cabpkv1.File, string, error) {
},
}, containerdApplyPatchesScriptOnRemoteCommand, nil
}

//nolint:gocritic // no need for named return values
func generateContainerdRestartScript() ([]cabpkv1.File, string) {
return []cabpkv1.File{
{
Path: containerdRestartScriptOnRemote,
Content: string(containerdRestartScript),
Permissions: "0700",
},
}, containerdRestartScriptOnRemoteCommand
}
16 changes: 0 additions & 16 deletions pkg/handlers/generic/mutation/mirrors/tests/generate_patches.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,17 +75,13 @@ func TestGeneratePatches(
gomega.HaveKeyWithValue(
"path", "/etc/containerd/apply-patches.sh",
),
gomega.HaveKeyWithValue(
"path", "/etc/containerd/restart.sh",
),
),
},
{
Operation: "add",
Path: "/spec/template/spec/kubeadmConfigSpec/preKubeadmCommands",
ValueMatcher: gomega.ContainElements(
"/bin/bash /etc/containerd/apply-patches.sh",
"/bin/bash /etc/containerd/restart.sh",
),
},
},
Expand Down Expand Up @@ -124,17 +120,13 @@ func TestGeneratePatches(
gomega.HaveKeyWithValue(
"path", "/etc/containerd/apply-patches.sh",
),
gomega.HaveKeyWithValue(
"path", "/etc/containerd/restart.sh",
),
),
},
{
Operation: "add",
Path: "/spec/template/spec/kubeadmConfigSpec/preKubeadmCommands",
ValueMatcher: gomega.ContainElements(
"/bin/bash /etc/containerd/apply-patches.sh",
"/bin/bash /etc/containerd/restart.sh",
),
},
},
Expand Down Expand Up @@ -173,17 +165,13 @@ func TestGeneratePatches(
gomega.HaveKeyWithValue(
"path", "/etc/containerd/apply-patches.sh",
),
gomega.HaveKeyWithValue(
"path", "/etc/containerd/restart.sh",
),
),
},
{
Operation: "add",
Path: "/spec/template/spec/preKubeadmCommands",
ValueMatcher: gomega.ContainElements(
"/bin/bash /etc/containerd/apply-patches.sh",
"/bin/bash /etc/containerd/restart.sh",
),
},
},
Expand Down Expand Up @@ -230,17 +218,13 @@ func TestGeneratePatches(
gomega.HaveKeyWithValue(
"path", "/etc/containerd/apply-patches.sh",
),
gomega.HaveKeyWithValue(
"path", "/etc/containerd/restart.sh",
),
),
},
{
Operation: "add",
Path: "/spec/template/spec/preKubeadmCommands",
ValueMatcher: gomega.ContainElements(
"/bin/bash /etc/containerd/apply-patches.sh",
"/bin/bash /etc/containerd/restart.sh",
),
},
},
Expand Down

0 comments on commit 3232a20

Please sign in to comment.