Skip to content

Commit

Permalink
use kind mapping to automatically set node registration
Browse files Browse the repository at this point in the history
  • Loading branch information
@fabriziopandini
fabriziopandini committed Jun 19, 2023
1 parent a90a241 commit fb82cdb
Show file tree
Hide file tree
Showing 5 changed files with 58 additions and 47 deletions.
8 changes: 8 additions & 0 deletions test/infrastructure/container/docker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ import (
"k8s.io/utils/pointer"

clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
"sigs.k8s.io/cluster-api/test/infrastructure/kind"
)

const (
Expand Down Expand Up @@ -390,6 +391,8 @@ func (d *dockerRuntime) RunContainer(ctx context.Context, runConfig *RunContaine
restartMaximumRetryCount = 1
}

// TODO: check if we can simplify the following code for the CAPD load balancer, which is now identified by runConfig.KindMode == kind.ModeNone

hostConfig := dockercontainer.HostConfig{
// Running containers in a container requires privileges.
// NOTE: we could try to replicate this with --cap-add, and use less
Expand All @@ -406,6 +409,11 @@ func (d *dockerRuntime) RunContainer(ctx context.Context, runConfig *RunContaine
}
networkConfig := network.NetworkingConfig{}

// NOTE: starting form Kind 1.20 kind requires CgroupnsMode to be set to private.
if runConfig.KindMode != kind.ModeNone && runConfig.KindMode != kind.Mode1_19 {
hostConfig.CgroupnsMode = "private"
}

if runConfig.IPFamily == clusterv1.IPv6IPFamily || runConfig.IPFamily == clusterv1.DualStackIPFamily {
hostConfig.Sysctls = map[string]string{
"net.ipv6.conf.all.disable_ipv6": "0",
Expand Down
3 changes: 3 additions & 0 deletions test/infrastructure/container/interface.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"io"

clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
"sigs.k8s.io/cluster-api/test/infrastructure/kind"
)

// providerKey is the key type for accessing the runtime provider in passed contexts.
Expand Down Expand Up @@ -98,6 +99,8 @@ type RunContainerInput struct {
// RestartPolicy to use for the container.
// If not set, defaults to "unless-stopped".
RestartPolicy string
// Defines how the kindest/node image must be started.
KindMode kind.Mode
}

// ExecContainerInput contains values for running exec on a container.
Expand Down
58 changes: 24 additions & 34 deletions test/infrastructure/docker/internal/docker/machine.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"strings"
"time"

"github.com/blang/semver"
"github.com/go-logr/logr"
"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
Expand All @@ -45,22 +46,17 @@ import (
"sigs.k8s.io/cluster-api/test/infrastructure/docker/internal/provisioning"
"sigs.k8s.io/cluster-api/test/infrastructure/docker/internal/provisioning/cloudinit"
"sigs.k8s.io/cluster-api/test/infrastructure/docker/internal/provisioning/ignition"
clusterapicontainer "sigs.k8s.io/cluster-api/util/container"
"sigs.k8s.io/cluster-api/test/infrastructure/kind"
"sigs.k8s.io/cluster-api/util/patch"
)

const (
defaultImageName = "kindest/node"
defaultImageTag = "v1.27.1"
)

var (
cloudProviderTaint = corev1.Taint{Key: "node.cloudprovider.kubernetes.io/uninitialized", Effect: corev1.TaintEffectNoSchedule}
)

type nodeCreator interface {
CreateControlPlaneNode(ctx context.Context, name, image, clusterName, listenAddress string, port int32, mounts []v1alpha4.Mount, portMappings []v1alpha4.PortMapping, labels map[string]string, ipFamily clusterv1.ClusterIPFamily) (node *types.Node, err error)
CreateWorkerNode(ctx context.Context, name, image, clusterName string, mounts []v1alpha4.Mount, portMappings []v1alpha4.PortMapping, labels map[string]string, ipFamily clusterv1.ClusterIPFamily) (node *types.Node, err error)
CreateControlPlaneNode(ctx context.Context, name, clusterName, listenAddress string, port int32, mounts []v1alpha4.Mount, portMappings []v1alpha4.PortMapping, labels map[string]string, ipFamily clusterv1.ClusterIPFamily, kindMapping kind.Mapping) (node *types.Node, err error)
CreateWorkerNode(ctx context.Context, name, clusterName string, mounts []v1alpha4.Mount, portMappings []v1alpha4.PortMapping, labels map[string]string, ipFamily clusterv1.ClusterIPFamily, kindMapping kind.Mapping) (node *types.Node, err error)
}

// Machine implement a service for managing the docker containers hosting a kubernetes nodes.
Expand Down Expand Up @@ -212,40 +208,54 @@ func (m *Machine) Create(ctx context.Context, image string, role string, version
if m.container == nil {
var err error

machineImage := m.machineImage(version)
// Get the kind kindMapping for the target K8s version.
// NOTE: The kind kindMapping allows to select the most recent kindest/node image available, if any, as well as
// provide info about the mode to be used when starting the kindest/node image itself.
if version == nil {
return errors.New("Cannot create a DockerMachine for a nil version")
}

semVer, err := semver.Parse(strings.TrimPrefix(*version, "v"))
if err != nil {
return errors.Wrap(err, "failed to parse DockerMachine version")
}

kindMapping := kind.GetMapping(semVer)

// Use custom image if provided.
if image != "" {
machineImage = image
kindMapping.Image = image
}

switch role {
case constants.ControlPlaneNodeRoleValue:
log.Info(fmt.Sprintf("Creating control plane machine container with image %s", machineImage))
log.Info(fmt.Sprintf("Creating control plane machine container with image %s", kindMapping.Image))
m.container, err = m.nodeCreator.CreateControlPlaneNode(
ctx,
m.ContainerName(),
machineImage,
m.cluster,
"127.0.0.1",
0,
kindMounts(mounts),
nil,
labels,
m.ipFamily,
kindMapping,
)
if err != nil {
return errors.WithStack(err)
}
case constants.WorkerNodeRoleValue:
log.Info(fmt.Sprintf("Creating worker machine container with image %s", machineImage))
log.Info(fmt.Sprintf("Creating worker machine container with image %s", kindMapping.Image))
m.container, err = m.nodeCreator.CreateWorkerNode(
ctx,
m.ContainerName(),
machineImage,
m.cluster,
kindMounts(mounts),
nil,
labels,
m.ipFamily,
kindMapping,
)
if err != nil {
return errors.WithStack(err)
Expand Down Expand Up @@ -527,26 +537,6 @@ func (m *Machine) Delete(ctx context.Context) error {
return nil
}

// machineImage is the image of the container node with the machine.
func (m *Machine) machineImage(version *string) string {
if version == nil {
defaultImage := fmt.Sprintf("%s:%s", defaultImageName, defaultImageTag)
return defaultImage
}

// TODO(fp) make this smarter
// - allows usage of custom docker repository & image names
// - add v only for semantic versions
versionString := *version
if !strings.HasPrefix(versionString, "v") {
versionString = fmt.Sprintf("v%s", versionString)
}

versionString = clusterapicontainer.SemverToOCIImageTag(versionString)

return fmt.Sprintf("%s:%s", defaultImageName, versionString)
}

func logContainerDebugInfo(ctx context.Context, log logr.Logger, name string) {
containerRuntime, err := container.RuntimeFrom(ctx)
if err != nil {
Expand Down
27 changes: 17 additions & 10 deletions ...re/docker/internal/docker/kind_manager.go → ...ructure/docker/internal/docker/manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
"sigs.k8s.io/cluster-api/test/infrastructure/container"
"sigs.k8s.io/cluster-api/test/infrastructure/docker/internal/docker/types"
"sigs.k8s.io/cluster-api/test/infrastructure/kind"
)

// KubeadmContainerPort is the port that kubeadm listens on in the container.
Expand All @@ -46,21 +47,21 @@ type Manager struct{}

type nodeCreateOpts struct {
Name string
Image string
ClusterName string
Role string
EntryPoint []string
Mounts []v1alpha4.Mount
PortMappings []v1alpha4.PortMapping
Labels map[string]string
IPFamily clusterv1.ClusterIPFamily
kindMapping kind.Mapping
}

// CreateControlPlaneNode will create a new control plane container.
// NOTE: If port is 0 picking a host port for the control plane is delegated to the container runtime and is not stable across container restarts.
// This means that connection to a control plane node may take some time to recover if the underlying container is restarted.
func (m *Manager) CreateControlPlaneNode(ctx context.Context, name, image, clusterName, listenAddress string, port int32, mounts []v1alpha4.Mount, portMappings []v1alpha4.PortMapping, labels map[string]string, ipFamily clusterv1.ClusterIPFamily) (*types.Node, error) {
// add api server port mapping
func (m *Manager) CreateControlPlaneNode(ctx context.Context, name, clusterName, listenAddress string, port int32, mounts []v1alpha4.Mount, portMappings []v1alpha4.PortMapping, labels map[string]string, ipFamily clusterv1.ClusterIPFamily, kindMapping kind.Mapping) (*types.Node, error) {
// add api server port kindMapping
portMappingsWithAPIServer := append(portMappings, v1alpha4.PortMapping{
ListenAddress: listenAddress,
HostPort: port,
Expand All @@ -69,13 +70,13 @@ func (m *Manager) CreateControlPlaneNode(ctx context.Context, name, image, clust
})
createOpts := &nodeCreateOpts{
Name: name,
Image: image,
ClusterName: clusterName,
Role: constants.ControlPlaneNodeRoleValue,
PortMappings: portMappingsWithAPIServer,
Mounts: mounts,
Labels: labels,
IPFamily: ipFamily,
kindMapping: kindMapping,
}
node, err := createNode(ctx, createOpts)
if err != nil {
Expand All @@ -86,16 +87,16 @@ func (m *Manager) CreateControlPlaneNode(ctx context.Context, name, image, clust
}

// CreateWorkerNode will create a new worker container.
func (m *Manager) CreateWorkerNode(ctx context.Context, name, image, clusterName string, mounts []v1alpha4.Mount, portMappings []v1alpha4.PortMapping, labels map[string]string, ipFamily clusterv1.ClusterIPFamily) (*types.Node, error) {
func (m *Manager) CreateWorkerNode(ctx context.Context, name, clusterName string, mounts []v1alpha4.Mount, portMappings []v1alpha4.PortMapping, labels map[string]string, ipFamily clusterv1.ClusterIPFamily, kindMapping kind.Mapping) (*types.Node, error) {
createOpts := &nodeCreateOpts{
Name: name,
Image: image,
ClusterName: clusterName,
Role: constants.WorkerNodeRoleValue,
PortMappings: portMappings,
Mounts: mounts,
Labels: labels,
IPFamily: ipFamily,
kindMapping: kindMapping,
}
return createNode(ctx, createOpts)
}
Expand All @@ -104,7 +105,7 @@ func (m *Manager) CreateWorkerNode(ctx context.Context, name, image, clusterName
// NOTE: If port is 0 picking a host port for the load balancer is delegated to the container runtime and is not stable across container restarts.
// This can break the Kubeconfig in kind, i.e. the file resulting from `kind get kubeconfig -n $CLUSTER_NAME' if the load balancer container is restarted.
func (m *Manager) CreateExternalLoadBalancerNode(ctx context.Context, name, image, clusterName, listenAddress string, port int32, _ clusterv1.ClusterIPFamily) (*types.Node, error) {
// load balancer port mapping
// load balancer port kindMapping
portMappings := []v1alpha4.PortMapping{{
ListenAddress: listenAddress,
HostPort: port,
Expand All @@ -113,11 +114,16 @@ func (m *Manager) CreateExternalLoadBalancerNode(ctx context.Context, name, imag
}}
createOpts := &nodeCreateOpts{
Name: name,
Image: image,
ClusterName: clusterName,
Role: constants.ExternalLoadBalancerNodeRoleValue,
PortMappings: portMappings,
EntryPoint: haproxyEntrypoint,
// Load balancer doesn't have an equivalent in kind, but we use a kind.Mapping to
// forward the image name to create node.
kindMapping: kind.Mapping{
Image: image,
Mode: kind.ModeNone,
},
}
node, err := createNode(ctx, createOpts)
if err != nil {
Expand All @@ -141,7 +147,7 @@ func createNode(ctx context.Context, opts *nodeCreateOpts) (*types.Node, error)

runOptions := &container.RunContainerInput{
Name: opts.Name, // make hostname match container name
Image: opts.Image,
Image: opts.kindMapping.Image,
Labels: containerLabels,
// runtime persistent storage
// this ensures that E.G. pods, logs etc. are not on the container
Expand All @@ -158,6 +164,7 @@ func createNode(ctx context.Context, opts *nodeCreateOpts) (*types.Node, error)
"/run": "", // systemd wants a writable /run
},
IPFamily: opts.IPFamily,
KindMode: opts.kindMapping.Mode,
}
if opts.Role == constants.ControlPlaneNodeRoleValue {
runOptions.EnvironmentVars = map[string]string{
Expand All @@ -177,7 +184,7 @@ func createNode(ctx context.Context, opts *nodeCreateOpts) (*types.Node, error)
return nil, err
}

return types.NewNode(opts.Name, opts.Image, opts.Role), nil
return types.NewNode(opts.Name, opts.kindMapping.Image, opts.Role), nil
}

func generateMountInfo(mounts []v1alpha4.Mount) []container.Mount {
Expand Down
9 changes: 6 additions & 3 deletions ...cker/internal/docker/kind_manager_test.go → ...re/docker/internal/docker/manager_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (

clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
"sigs.k8s.io/cluster-api/test/infrastructure/container"
"sigs.k8s.io/cluster-api/test/infrastructure/kind"
)

func TestCreateNode(t *testing.T) {
Expand All @@ -44,12 +45,14 @@ func TestCreateNode(t *testing.T) {
}
createOpts := &nodeCreateOpts{
Name: "TestName",
Image: "TestImage",
ClusterName: "TestClusterName",
Role: constants.ControlPlaneNodeRoleValue,
PortMappings: portMappingsWithAPIServer,
Mounts: []v1alpha4.Mount{},
IPFamily: clusterv1.IPv4IPFamily,
kindMapping: kind.Mapping{
Image: "TestImage",
},
}
_, err := createNode(ctx, createOpts)

Expand All @@ -76,7 +79,7 @@ func TestCreateControlPlaneNode(t *testing.T) {

containerRuntime.ResetRunContainerCallLogs()
m := Manager{}
node, err := m.CreateControlPlaneNode(ctx, "TestName", "TestImage", "TestCluster", "100.100.100.100", 80, []v1alpha4.Mount{}, []v1alpha4.PortMapping{}, make(map[string]string), clusterv1.IPv4IPFamily)
node, err := m.CreateControlPlaneNode(ctx, "TestName", "TestCluster", "100.100.100.100", 80, []v1alpha4.Mount{}, []v1alpha4.PortMapping{}, make(map[string]string), clusterv1.IPv4IPFamily, kind.Mapping{Image: "TestImage"})

g.Expect(err).ShouldNot(HaveOccurred())
g.Expect(node.Role()).Should(Equal(constants.ControlPlaneNodeRoleValue))
Expand All @@ -99,7 +102,7 @@ func TestCreateWorkerNode(t *testing.T) {

containerRuntime.ResetRunContainerCallLogs()
m := Manager{}
node, err := m.CreateWorkerNode(ctx, "TestName", "TestImage", "TestCluster", []v1alpha4.Mount{}, []v1alpha4.PortMapping{}, make(map[string]string), clusterv1.IPv4IPFamily)
node, err := m.CreateWorkerNode(ctx, "TestName", "TestCluster", []v1alpha4.Mount{}, []v1alpha4.PortMapping{}, make(map[string]string), clusterv1.IPv4IPFamily, kind.Mapping{Image: "TestImage"})

g.Expect(err).ShouldNot(HaveOccurred())
g.Expect(node.Role()).Should(Equal(constants.WorkerNodeRoleValue))
Expand Down

0 comments on commit fb82cdb

Please sign in to comment.