Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

🌱 [POC] [DNR] Test for adding mp to quickstart templates #9440

Closed
wants to merge 4 commits into from
Closed

🌱 [POC] [DNR] Test for adding mp to quickstart templates #9440

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
68c8891
Add mp to quickstart templates
willie-yao Sep 11, 2023
cb22353
Reviews
willie-yao Sep 12, 2023
3efb573
dump all pods
killianmuldoon Sep 15, 2023
4bc3ba3
Patch for MachinePool preLoadImages
killianmuldoon Sep 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions docs/book/src/reference/owner_references.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ These owner references are almost all tested in an [end-to-end test](https://git
|----------------------------|--------------------|------------|--------------------------|
| ClusterResourcesSet | None | | |
| ClusterResourcesSetBinding | ClusterResourceSet | no | May have many CRS owners |
| MachinePool | Cluster | unknown | Not tested in e2e |
| MachinePool | Cluster | no | |


## KubeadmControlPlane types
Expand All @@ -65,7 +65,7 @@ These owner references are almost all tested in an [end-to-end test](https://git
| type | Owner | Controller | Note |
|-----------------------|--------------|------------|-------------------------------------------------|
| KubeadmConfig | Machine | yes | When created for Machine |
| KubeadmConfig | MachinePool | unknown | When created for MachinePool. Not tested in e2e |
| KubeadmConfig | MachinePool | yes | When created for MachinePool |
| KubeadmConfigTemplate | Cluster | no | When referenced in MachineDeployment spec |
| KubeadmConfigTemplate | ClusterClass | no | When referenced in ClusterClass |

Expand All @@ -77,6 +77,6 @@ These owner references are almost all tested in an [end-to-end test](https://git
| InfrastructureMachineTemplate | ClusterClass | no | When referenced in a ClusterClass |
| InfrastructureCluster | Cluster | yes | |
| InfrastructureClusterTemplate | ClusterClass | no | |
| InfrastructureMachinePool | MachinePool | unknown | Not tested in e2e |
| InfrastructureMachinePool | MachinePool | yes | |


16 changes: 16 additions & 0 deletions test/e2e/data/infrastructure-docker/main/bases/cluster-with-topology.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,22 @@ spec:
rollingUpdate:
maxSurge: "20%"
maxUnavailable: 0
machinePools:
- class: "default-worker"
name: "mp-0"
metadata:
labels:
Cluster.topology.machinePool.label: "Cluster.topology.machinePool.labelValue"
# Note: this label is propagated to Nodes.
Cluster.topology.machinePool.label.node.cluster.x-k8s.io: "Cluster.topology.machinePool.nodeLabelValue"
annotations:
Cluster.topology.machinePool.annotation: "Cluster.topology.machinePool.annotationValue"
nodeDeletionTimeout: "30s"
nodeVolumeDetachTimeout: "5m"
minReadySeconds: 5
replicas: ${WORKER_MACHINE_COUNT}
failureDomains:
- fd4
variables:
# We set an empty value to use the default tag kubeadm init is using.
- name: etcdImageTag
Expand Down
27 changes: 27 additions & 0 deletions test/e2e/data/infrastructure-docker/main/clusterclass-quick-start-runtimesdk.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,21 @@ spec:
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: DockerMachineTemplate
name: quick-start-default-worker-machinetemplate
machinePools:
- class: default-worker
namingStrategy:
template: "{{ .cluster.name }}-mp-{{ .machinePool.topologyName }}-{{ .random }}"
template:
bootstrap:
ref:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
name: quick-start-default-worker-bootstraptemplate
infrastructure:
ref:
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: DockerMachinePoolTemplate
name: quick-start-default-worker-machinepooltemplate
variables:
# This variable is not used in any patch, but is here to ensure the variable discovered from the runtime hook is correctly
# used.
Expand Down Expand Up @@ -122,6 +137,18 @@ spec:
- containerPath: "/var/run/docker.sock"
hostPath: "/var/run/docker.sock"
---
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: DockerMachinePoolTemplate
metadata:
name: quick-start-default-worker-machinepooltemplate
spec:
template:
spec:
template:
extraMounts:
- containerPath: "/var/run/docker.sock"
hostPath: "/var/run/docker.sock"
---
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
metadata:
Expand Down
85 changes: 83 additions & 2 deletions test/e2e/data/infrastructure-docker/main/clusterclass-quick-start.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ spec:
ref:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
name: quick-start-default-worker-bootstraptemplate
name: quick-start-md-default-worker-bootstraptemplate
infrastructure:
ref:
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
Expand All @@ -54,6 +54,24 @@ spec:
- type: e2e.remediation.condition
status: "False"
timeout: 20s
machinePools:
- class: default-worker
template:
metadata:
labels:
ClusterClass.machinePool.label: "ClusterClass.machinePool.labelValue"
annotations:
ClusterClass.machinePool.annotation: "ClusterClass.machinePool.annotationValue"
bootstrap:
ref:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
name: quick-start-mp-default-worker-bootstraptemplate
infrastructure:
ref:
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: DockerMachinePoolTemplate
name: quick-start-default-worker-machinepooltemplate
variables:
- name: lbImageRepository
required: true
Expand Down Expand Up @@ -182,6 +200,19 @@ spec:
valueFrom:
template: |
kindest/node:{{ .builtin.controlPlane.version | replace "+" "_" }}
- selector:
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: DockerMachinePoolTemplate
matchResources:
machinePoolClass:
names:
- default-worker
jsonPatches:
- op: add
path: "/spec/template/spec/template/customImage"
valueFrom:
template: |
kindest/node:{{ .builtin.machinePool.version | replace "+" "_" }}
- name: preloadImages
description: |
Sets the container images to preload to the node that is used for running dockerMachines.
Expand All @@ -201,6 +232,7 @@ spec:
path: "/spec/template/spec/preLoadImages"
valueFrom:
variable: preLoadImages

- name: kubeadmControlPlaneMaxSurge
description: "Sets the maxSurge value used for rolloutStrategy in the KubeadmControlPlane."
enabledIf: '{{ ne .kubeadmControlPlaneMaxSurge "" }}'
Expand Down Expand Up @@ -264,6 +296,18 @@ spec:
path: "/spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs"
value:
cloud-provider: "external"
- selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machinePoolClass:
names:
- '*-worker'
jsonPatches:
- op: add
path: "/spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs"
value:
cloud-provider: "external"
- name: localEndpointIPv6
enabledIf: "{{ .ipv6Primary }}"
description: "Configures KCP to use IPv6 for its localAPIEndpoint."
Expand Down Expand Up @@ -440,10 +484,26 @@ spec:
- containerPath: "/var/run/docker.sock"
hostPath: "/var/run/docker.sock"
---
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: DockerMachinePoolTemplate
metadata:
name: quick-start-default-worker-machinepooltemplate
labels:
InfraMachinePoolTemplate.machinePool.label: "InfraMachinePoolTemplate.machinePool.labelValue"
annotations:
InfraMachinePoolTemplate.machinePool.annotation: "InfraMachinePoolTemplate.machinePool.annotationValue"
spec:
template:
spec:
template:
extraMounts:
- containerPath: "/var/run/docker.sock"
hostPath: "/var/run/docker.sock"
---
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
metadata:
name: quick-start-default-worker-bootstraptemplate
name: quick-start-md-default-worker-bootstraptemplate
labels:
BootstrapConfigTemplate.machineDeployment.label: "BootstrapConfigTemplate.machineDeployment.labelValue"
annotations:
Expand All @@ -460,3 +520,24 @@ spec:
nodeRegistration: # node registration parameters are automatically injected by CAPD according to the kindest/node image in use.
kubeletExtraArgs: # having a not empty kubeletExtraArgs is required for the externalCloudProvider to work
eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
---
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
metadata:
name: quick-start-mp-default-worker-bootstraptemplate
labels:
BootstrapConfigTemplate.machinePool.template.label: "BootstrapConfigTemplate.machinePool.template.labelValue"
annotations:
BootstrapConfigTemplate.machinePool.template.annotation: "BootstrapConfigTemplate.machinePool.template.annotationValue"
spec:
template:
metadata:
labels:
BootstrapConfigTemplate.machinePool.template.label: "BootstrapConfigTemplate.machinePool.template.labelValue"
annotations:
BootstrapConfigTemplate.machinePool.template.annotation: "BootstrapConfigTemplate.machinePool.template.annotationValue"
spec:
joinConfiguration:
nodeRegistration: # node registration parameters are automatically injected by CAPD according to the kindest/node image in use.
kubeletExtraArgs: # having a not empty kubeletExtraArgs is required for the externalCloudProvider to work
eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
2 changes: 1 addition & 1 deletion test/e2e/quick_start_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ var _ = Describe("When following the Cluster API quick-start", func() {
})
})

var _ = Describe("When following the Cluster API quick-start with ClusterClass [PR-Blocking] [ClusterClass]", func() {
var _ = Describe("When following the Cluster API quick-start with ClusterClass [ClusterClass]", func() {
QuickStartSpec(ctx, func() QuickStartSpecInput {
return QuickStartSpecInput{
E2EConfig: e2eConfig,
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/self_hosted_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ var _ = Describe("When testing Cluster API working on self-hosted clusters", fun
})
})

var _ = Describe("When testing Cluster API working on self-hosted clusters using ClusterClass [ClusterClass]", func() {
var _ = Describe("When testing Cluster API working on self-hosted clusters using ClusterClass [PR-Blocking] [ClusterClass]", func() {
SelfHostedSpec(ctx, func() SelfHostedSpecInput {
return SelfHostedSpecInput{
E2EConfig: e2eConfig,
Expand Down
2 changes: 1 addition & 1 deletion test/framework/alltypes_helpers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ func DumpKubeSystemPodsForCluster(ctx context.Context, input DumpKubeSystemPodsF
podList.SetKind("Pod")
var listErr error
_ = wait.PollUntilContextTimeout(ctx, retryableOperationInterval, retryableOperationTimeout, true, func(ctx context.Context) (bool, error) {
if listErr = input.Lister.List(ctx, podList, client.InNamespace(metav1.NamespaceSystem)); listErr != nil {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@sbueringer @chrischdi WDYT about making a change like this anyway? We could just go through the known CAPI namespaces, but it makes sense to me to just dump all the pods in the workload clusters.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah let's just dump all Pods. Feel free to open a separate PR. I just approved the collecting Nodes PR so you can built it on top of it.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

aggree, dumping all pods would be a great thing esepcially for self-hosted tests :-)

if listErr = input.Lister.List(ctx, podList); listErr != nil {
return false, nil //nolint:nilerr
}
return true, nil
Expand Down
22 changes: 13 additions & 9 deletions test/framework/ownerreference_helpers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,6 @@ var (
clusterResourceSetBindingKind = "ClusterResourceSetBinding"
machinePoolKind = "MachinePool"

machinePoolOwner = metav1.OwnerReference{Kind: machinePoolKind, APIVersion: expv1.GroupVersion.String()}
machinePoolController = metav1.OwnerReference{Kind: machinePoolKind, APIVersion: expv1.GroupVersion.String(), Controller: pointer.Bool(true)}

clusterResourceSetOwner = metav1.OwnerReference{Kind: clusterResourceSetKind, APIVersion: addonsv1.GroupVersion.String()}
Expand Down Expand Up @@ -280,11 +279,12 @@ var KubeadmBootstrapOwnerReferenceAssertions = map[string]func([]metav1.OwnerRef

// Kinds for types in the Docker infrastructure package.
var (
dockerMachineKind = "DockerMachine"
dockerMachineTemplateKind = "DockerMachineTemplate"
dockerMachinePoolKind = "DockerMachinePool"
dockerClusterKind = "DockerCluster"
dockerClusterTemplateKind = "DockerClusterTemplate"
dockerMachineKind = "DockerMachine"
dockerMachineTemplateKind = "DockerMachineTemplate"
dockerMachinePoolKind = "DockerMachinePool"
dockerMachinePoolTemplateKind = "DockerMachinePoolTemplate"
dockerClusterKind = "DockerCluster"
dockerClusterTemplateKind = "DockerClusterTemplate"
)

// DockerInfraOwnerReferenceAssertions maps Docker Infrastructure types to functions which return an error if the passed
Expand All @@ -310,8 +310,12 @@ var DockerInfraOwnerReferenceAssertions = map[string]func([]metav1.OwnerReferenc
return HasExactOwners(owners, clusterClassOwner)
},
dockerMachinePoolKind: func(owners []metav1.OwnerReference) error {
// DockerMachinePool must be owned by a MachinePool.
return HasExactOwners(owners, machinePoolOwner)
// DockerMachinePool must be owned and controlled by a MachinePool.
return HasExactOwners(owners, machinePoolController)
},
dockerMachinePoolTemplateKind: func(owners []metav1.OwnerReference) error {
// DockerMachinePoolTemplate must be owned by a ClusterClass.
return HasExactOwners(owners, clusterClassOwner)
},
}

Expand All @@ -328,7 +332,7 @@ func HasExactOwners(gotOwners []metav1.OwnerReference, wantOwners ...metav1.Owne
sort.Strings(wantComparable)

if !reflect.DeepEqual(gotComparable, wantComparable) {
return fmt.Errorf("wanted %v, actual %v", wantOwners, gotOwners)
return fmt.Errorf("wanted %v, actual %v", wantComparable, gotComparable)
}
return nil
}
Expand Down
Loading