diff --git a/Makefile b/Makefile index fbd59844d..1aa2e26ab 100644 --- a/Makefile +++ b/Makefile @@ -225,6 +225,7 @@ e2e-test-templates-v1beta1: $(KUSTOMIZE) ## Generate cluster templates for v1bet mkdir -p $(TEST_TEMPLATES_TARGET_DIR)/v1beta1/ $(KUSTOMIZE) build $(REPO_ROOT)/templates/experimental-crs-cni --load-restrictor LoadRestrictionsNone > $(TEST_TEMPLATES_TARGET_DIR)/v1beta1/cluster-template.yaml $(KUSTOMIZE) build $(REPO_ROOT)/templates/experimental-kube-vip-crs-cni --load-restrictor LoadRestrictionsNone > $(TEST_TEMPLATES_TARGET_DIR)/v1beta1/cluster-template-kube-vip.yaml + $(KUSTOMIZE) build $(REPO_ROOT)/templates/experimental-emlb-crs-cni --load-restrictor LoadRestrictionsNone > $(TEST_TEMPLATES_TARGET_DIR)/v1beta1/cluster-template-emlb.yaml $(KUSTOMIZE) build $(REPO_ROOT)/test/e2e/data/v1beta1/cluster-template-kcp-scale-in --load-restrictor LoadRestrictionsNone > $(TEST_TEMPLATES_TARGET_DIR)/v1beta1/cluster-template-kcp-scale-in.yaml $(KUSTOMIZE) build $(REPO_ROOT)/test/e2e/data/v1beta1/cluster-template-node-drain --load-restrictor LoadRestrictionsNone > $(TEST_TEMPLATES_TARGET_DIR)/v1beta1/cluster-template-node-drain.yaml $(KUSTOMIZE) build $(REPO_ROOT)/test/e2e/data/v1beta1/cluster-template-md-remediation --load-restrictor LoadRestrictionsNone > $(TEST_TEMPLATES_TARGET_DIR)/v1beta1/cluster-template-md-remediation.yaml @@ -282,6 +283,8 @@ generate: ## Generate code .PHONY: generate-templates generate-templates: $(KUSTOMIZE) ## Generate cluster templates + $(KUSTOMIZE) build templates/experimental-emlb --load-restrictor LoadRestrictionsNone > templates/cluster-template-emlb.yaml + $(KUSTOMIZE) build templates/experimental-emlb-crs-cni --load-restrictor LoadRestrictionsNone > templates/cluster-template-emlb-crs-cni.yaml $(KUSTOMIZE) build templates/experimental-kube-vip-crs-cni --load-restrictor LoadRestrictionsNone > templates/cluster-template-kube-vip-crs-cni.yaml $(KUSTOMIZE) build templates/experimental-kube-vip --load-restrictor LoadRestrictionsNone > templates/cluster-template-kube-vip.yaml $(KUSTOMIZE) build templates/experimental-crs-cni --load-restrictor LoadRestrictionsNone > templates/cluster-template-crs-cni.yaml diff --git a/api/v1beta1/packetcluster_types.go b/api/v1beta1/packetcluster_types.go index 735d4a889..43d554f48 100644 --- a/api/v1beta1/packetcluster_types.go +++ b/api/v1beta1/packetcluster_types.go @@ -27,6 +27,12 @@ const ( ClusterFinalizer = "packetcluster.infrastructure.cluster.x-k8s.io" // NetworkInfrastructureReadyCondition reports of current status of cluster infrastructure. NetworkInfrastructureReadyCondition clusterv1.ConditionType = "NetworkInfrastructureReady" + // EMLBVIPID is the string used to refer to the EMLB load balancer and VIP Manager type. + EMLBVIPID = "EMLB" + // CPEMID is the string used to refer to the CPEM load balancer and VIP Manager type. + CPEMID = "CPEM" + // KUBEVIPID is the string used to refer to the Kube VIP load balancer and VIP Manager type. + KUBEVIPID = "KUBE_VIP" ) // VIPManagerType describes if the VIP will be managed by CPEM or kube-vip or Equinix Metal Load Balancer. diff --git a/controllers/packetcluster_controller.go b/controllers/packetcluster_controller.go index 1bd0d5bf7..1f85c9b7c 100644 --- a/controllers/packetcluster_controller.go +++ b/controllers/packetcluster_controller.go @@ -125,7 +125,7 @@ func (r *PacketClusterReconciler) reconcileNormal(ctx context.Context, clusterSc packetCluster := clusterScope.PacketCluster switch { - case packetCluster.Spec.VIPManager == emlb.EMLBVIPID: + case packetCluster.Spec.VIPManager == infrav1.EMLBVIPID: if !packetCluster.Spec.ControlPlaneEndpoint.IsValid() { // Create new EMLB object lb := emlb.NewEMLB(r.PacketClient.GetConfig().DefaultHeader["X-Auth-Token"], packetCluster.Spec.ProjectID, packetCluster.Spec.Metro) @@ -135,7 +135,7 @@ func (r *PacketClusterReconciler) reconcileNormal(ctx context.Context, clusterSc return err } } - case packetCluster.Spec.VIPManager == "KUBE_VIP": + case packetCluster.Spec.VIPManager == infrav1.KUBEVIPID: log.Info("KUBE_VIP VIPManager Detected") if err := r.PacketClient.EnableProjectBGP(ctx, packetCluster.Spec.ProjectID); err != nil { log.Error(err, "error enabling bgp for project") @@ -143,7 +143,7 @@ func (r *PacketClusterReconciler) reconcileNormal(ctx context.Context, clusterSc } } - if packetCluster.Spec.VIPManager != emlb.EMLBVIPID { + if packetCluster.Spec.VIPManager != infrav1.EMLBVIPID { ipReserv, err := r.PacketClient.GetIPByClusterIdentifier(ctx, clusterScope.Namespace(), clusterScope.Name(), packetCluster.Spec.ProjectID) switch { case errors.Is(err, packet.ErrControlPlanEndpointNotFound): @@ -192,7 +192,7 @@ func (r *PacketClusterReconciler) reconcileDelete(ctx context.Context, clusterSc packetCluster := clusterScope.PacketCluster - if packetCluster.Spec.VIPManager == emlb.EMLBVIPID { + if packetCluster.Spec.VIPManager == infrav1.EMLBVIPID { // Create new EMLB object lb := emlb.NewEMLB(r.PacketClient.GetConfig().DefaultHeader["X-Auth-Token"], packetCluster.Spec.ProjectID, packetCluster.Spec.Metro) diff --git a/controllers/packetmachine_controller.go b/controllers/packetmachine_controller.go index 209aedac3..707fbb34d 100644 --- a/controllers/packetmachine_controller.go +++ b/controllers/packetmachine_controller.go @@ -353,8 +353,8 @@ func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *s var controlPlaneEndpointAddress string var cpemLBConfig string var emlbID string - switch { - case machineScope.PacketCluster.Spec.VIPManager == "CPEM": + switch machineScope.PacketCluster.Spec.VIPManager { + case infrav1.CPEMID, infrav1.KUBEVIPID: controlPlaneEndpoint, _ = r.PacketClient.GetIPByClusterIdentifier( ctx, machineScope.Cluster.Namespace, @@ -368,7 +368,7 @@ func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *s addrs = append(addrs, a) } controlPlaneEndpointAddress = controlPlaneEndpoint.GetAddress() - case machineScope.PacketCluster.Spec.VIPManager == emlb.EMLBVIPID: + case infrav1.EMLBVIPID: controlPlaneEndpointAddress = machineScope.Cluster.Spec.ControlPlaneEndpoint.Host cpemLBConfig = "emlb:///" + machineScope.PacketCluster.Spec.Metro emlbID = machineScope.PacketCluster.Annotations["equinix.com/loadbalancerID"] @@ -404,7 +404,7 @@ func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *s machineScope.SetProviderID(dev.GetId()) machineScope.SetInstanceStatus(infrav1.PacketResourceStatus(dev.GetState())) - if machineScope.PacketCluster.Spec.VIPManager == "KUBE_VIP" { + if machineScope.PacketCluster.Spec.VIPManager == infrav1.KUBEVIPID { if err := r.PacketClient.EnsureNodeBGPEnabled(ctx, dev.GetId()); err != nil { // Do not treat an error enabling bgp on machine as fatal return ctrl.Result{RequeueAfter: time.Second * 20}, fmt.Errorf("failed to enable bgp on machine %s: %w", machineScope.Name(), err) @@ -426,7 +426,7 @@ func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *s log.Info("Machine instance is active", "instance-id", machineScope.ProviderID()) switch { - case machineScope.PacketCluster.Spec.VIPManager == "CPEM": + case machineScope.PacketCluster.Spec.VIPManager == infrav1.CPEMID: controlPlaneEndpoint, _ = r.PacketClient.GetIPByClusterIdentifier( ctx, machineScope.Cluster.Namespace, @@ -441,7 +441,7 @@ func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *s return ctrl.Result{RequeueAfter: time.Second * 20}, nil } } - case machineScope.PacketCluster.Spec.VIPManager == emlb.EMLBVIPID: + case machineScope.PacketCluster.Spec.VIPManager == infrav1.EMLBVIPID: if machineScope.IsControlPlane() { // Create new EMLB object lb := emlb.NewEMLB(r.PacketClient.GetConfig().DefaultHeader["X-Auth-Token"], machineScope.PacketCluster.Spec.ProjectID, machineScope.PacketCluster.Spec.Metro) @@ -543,12 +543,14 @@ func (r *PacketMachineReconciler) reconcileDelete(ctx context.Context, machineSc return fmt.Errorf("%w: %s", errMissingDevice, packetmachine.Name) } - if machineScope.PacketCluster.Spec.VIPManager == emlb.EMLBVIPID { - // Create new EMLB object - lb := emlb.NewEMLB(r.PacketClient.GetConfig().DefaultHeader["X-Auth-Token"], machineScope.PacketCluster.Spec.ProjectID, packetmachine.Spec.Metro) + if machineScope.PacketCluster.Spec.VIPManager == infrav1.EMLBVIPID { + if machineScope.IsControlPlane() { + // Create new EMLB object + lb := emlb.NewEMLB(r.PacketClient.GetConfig().DefaultHeader["X-Auth-Token"], machineScope.PacketCluster.Spec.ProjectID, packetmachine.Spec.Metro) - if err := lb.DeleteLoadBalancerOrigin(ctx, machineScope); err != nil { - return fmt.Errorf("failed to delete load balancer origin: %w", err) + if err := lb.DeleteLoadBalancerOrigin(ctx, machineScope); err != nil { + return fmt.Errorf("failed to delete load balancer origin: %w", err) + } } } diff --git a/docs/experiences/flavors.md b/docs/experiences/flavors.md index 91db88cf4..61770dc07 100644 --- a/docs/experiences/flavors.md +++ b/docs/experiences/flavors.md @@ -4,15 +4,29 @@ ### API Server VIP Management Choice -By default CPEM will be used to manage the EIP that serves as the VIP for the api-server. As of v0.6.0 you can choose to use kube-vip to manage the api-server VIP instead of CPEM. +By default CPEM will be used to manage the EIP that serves as the VIP for the api-server. Other flavors include kube-vip and Equinix Metal Load Balancer. + +### Choosing Equinix Metal Load Balancer + +To use Equinix Metal Load Balancer, when generating the template with `clusterctl`, pass in the `--flavor emlb` flag. For example, your `clusterctl generate` command might look like the following: + +```sh +clusterctl generate cluster capi-quickstart \ + --kubernetes-version v1.31.0 \ + --control-plane-machine-count=3 \ + --worker-machine-count=3 \ + --infrastructure packet \ + --flavor emlb + > capi-quickstart.yaml +``` ### Choosing Kube-VIP - To use kube-vip, when generating the template with `clusterctl`, pass in the `--flavor kube-vip` flag. For example, your `clusterctl generate` command might look like the following: +To use kube-vip, when generating the template with `clusterctl`, pass in the `--flavor kube-vip` flag. For example, your `clusterctl generate` command might look like the following: ```sh clusterctl generate cluster capi-quickstart \ - --kubernetes-version v1.24.0 \ + --kubernetes-version v1.31.0 \ --control-plane-machine-count=3 \ --worker-machine-count=3 \ --infrastructure packet \ diff --git a/internal/emlb/emlb.go b/internal/emlb/emlb.go index 3c1eccab6..474144028 100644 --- a/internal/emlb/emlb.go +++ b/internal/emlb/emlb.go @@ -49,8 +49,6 @@ const ( loadBalancerPoolIDAnnotation = "equinix.com/loadbalancerpoolID" // loadBalancerPoolOriginIDAnnotation is the anotation key representing the origin ID of a PacketMachine. loadBalancerOriginIDAnnotation = "equinix.com/loadbalanceroriginID" - // EMLBVIPID is the stringused to refer to the EMLB load balancer and VIP Manager type. - EMLBVIPID = "EMLB" // loadbalancerTokenExchangeURL is the default URL to use for Token Exchange to talk to the Equinix Metal Load Balancer API. loadbalancerTokenExchnageURL = "https://iam.metalctrl.io/api-keys/exchange" //nolint:gosec ) diff --git a/templates/cluster-template-emlb-crs-cni.yaml b/templates/cluster-template-emlb-crs-cni.yaml new file mode 100644 index 000000000..df0674508 --- /dev/null +++ b/templates/cluster-template-emlb-crs-cni.yaml @@ -0,0 +1,259 @@ +apiVersion: v1 +data: ${CNI_RESOURCES} +kind: ConfigMap +metadata: + name: ${CLUSTER_NAME}-crs-cni +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-crs-cni +spec: + clusterSelector: + matchLabels: + cni: ${CLUSTER_NAME}-crs-cni + resources: + - kind: ConfigMap + name: ${CLUSTER_NAME}-crs-cni + strategy: ApplyOnce +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-worker-a +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + provider-id: equinixmetal://{{ `{{ v1.instance_id }}` }} + preKubeadmCommands: + - | + sed -ri '/\sswap\s/s/^#?/#/' /etc/fstab + swapoff -a + mount -a + cat < /etc/modules-load.d/containerd.conf + overlay + br_netfilter + EOF + modprobe overlay + modprobe br_netfilter + cat < /etc/sysctl.d/99-kubernetes-cri.conf + net.bridge.bridge-nf-call-iptables = 1 + net.ipv4.ip_forward = 1 + net.bridge.bridge-nf-call-ip6tables = 1 + EOF + sysctl --system + export DEBIAN_FRONTEND=noninteractive + apt-get update -y + apt-get remove -y docker docker-engine containerd runc + apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq + install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg + MINOR_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | cut -d. -f1-2 ) + curl -fsSL https://pkgs.k8s.io/core:/stable:/$${MINOR_KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + chmod a+r /etc/apt/keyrings/docker.gpg + chmod a+r /etc/apt/keyrings/kubernetes-archive-keyring.gpg + echo "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" > /etc/apt/sources.list.d/docker.list + echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$${MINOR_KUBERNETES_VERSION}/deb/ /" > /etc/apt/sources.list.d/kubernetes.list + apt-get update -y + TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//') + RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1) + apt-get install -y containerd.io kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION} + cat < /etc/crictl.yaml + runtime-endpoint: unix:///run/containerd/containerd.sock + image-endpoint: unix:///run/containerd/containerd.sock + EOF + containerd config default > /etc/containerd/config.toml + sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml + sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml + systemctl restart containerd +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cni: ${CLUSTER_NAME}-crs-cni + name: ${CLUSTER_NAME} +spec: + clusterNetwork: + pods: + cidrBlocks: + - ${POD_CIDR:=192.168.0.0/16} + services: + cidrBlocks: + - ${SERVICE_CIDR:=172.26.0.0/16} + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: PacketCluster + name: ${CLUSTER_NAME} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + labels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + pool: worker-a + name: ${CLUSTER_NAME}-worker-a +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + pool: worker-a + template: + metadata: + labels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + pool: worker-a + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-worker-a + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: PacketMachineTemplate + name: ${CLUSTER_NAME}-worker-a + version: ${KUBERNETES_VERSION} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-provider: external + controllerManager: + extraArgs: + cloud-provider: external + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + provider-id: equinixmetal://{{ `{{ v1.instance_id }}` }} + joinConfiguration: + nodeRegistration: + ignorePreflightErrors: + - DirAvailable--etc-kubernetes-manifests + kubeletExtraArgs: + cloud-provider: external + provider-id: equinixmetal://{{ `{{ v1.instance_id }}` }} + postKubeadmCommands: + - |- + mkdir -p $HOME/.kube + cp /etc/kubernetes/admin.conf $HOME/.kube/config + echo "source <(kubectl completion bash)" >> $HOME/.bashrc + echo "alias k=kubectl" >> $HOME/.bashrc + echo "complete -o default -F __start_kubectl k" >> $HOME/.bashrc + if [ -f "/run/kubeadm/kubeadm.yaml" ]; then + export KUBECONFIG=/etc/kubernetes/admin.conf + export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/${CPEM_VERSION:=v3.8.0}/deployment.yaml + export SECRET_DATA='cloud-sa.json=''{"apiKey": "{{ .apiKey }}","projectID": "${PROJECT_ID}"}''' + kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}" || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}") || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}") + kubectl apply -f $${CPEM_YAML} || (sleep 1 && kubectl apply -f $${CPEM_YAML}) || (sleep 1 && kubectl apply -f $${CPEM_YAML}) + fi + preKubeadmCommands: + - | + sed -ri '/\sswap\s/s/^#?/#/' /etc/fstab + swapoff -a + mount -a + cat < /etc/modules-load.d/containerd.conf + overlay + br_netfilter + EOF + modprobe overlay + modprobe br_netfilter + cat < /etc/sysctl.d/99-kubernetes-cri.conf + net.bridge.bridge-nf-call-iptables = 1 + net.ipv4.ip_forward = 1 + net.bridge.bridge-nf-call-ip6tables = 1 + EOF + sysctl --system + export DEBIAN_FRONTEND=noninteractive + apt-get update -y + apt-get remove -y docker docker-engine containerd runc + apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq + major_vers=$(lsb_release -r | awk '{ print $2 }' | cut -d. -f1) + if [ "$major_vers" -ge 20 ]; then + apt-get install -y kubetail + fi + install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg + MINOR_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | cut -d. -f1-2 ) + curl -fsSL https://pkgs.k8s.io/core:/stable:/$${MINOR_KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + chmod a+r /etc/apt/keyrings/docker.gpg + chmod a+r /etc/apt/keyrings/kubernetes-archive-keyring.gpg + echo "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" > /etc/apt/sources.list.d/docker.list + echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$${MINOR_KUBERNETES_VERSION}/deb/ /" > /etc/apt/sources.list.d/kubernetes.list + apt-get update -y + TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//') + RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1) + apt-get install -y containerd.io kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION} + containerd config default > /etc/containerd/config.toml + cat < /etc/crictl.yaml + runtime-endpoint: unix:///run/containerd/containerd.sock + image-endpoint: unix:///run/containerd/containerd.sock + EOF + sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml + sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml + systemctl restart containerd + if [ -f "/run/kubeadm/kubeadm.yaml" ]; then + ip addr add {{ .controlPlaneEndpoint }} dev lo + fi + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: PacketMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: PacketCluster +metadata: + name: ${CLUSTER_NAME} +spec: + metro: ${METRO} + projectID: ${PROJECT_ID} + vipManager: EMLB +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: PacketMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + template: + spec: + billingCycle: hourly + machineType: ${CONTROLPLANE_NODE_TYPE} + os: ${NODE_OS:=ubuntu_20_04} + sshKeys: + - ${SSH_KEY} + tags: [] +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: PacketMachineTemplate +metadata: + name: ${CLUSTER_NAME}-worker-a +spec: + template: + spec: + billingCycle: hourly + machineType: ${WORKER_NODE_TYPE} + os: ${NODE_OS:=ubuntu_20_04} + sshKeys: + - ${SSH_KEY} + tags: [] diff --git a/templates/cluster-template-emlb.yaml b/templates/cluster-template-emlb.yaml index c7de6c204..6c8f02ee5 100644 --- a/templates/cluster-template-emlb.yaml +++ b/templates/cluster-template-emlb.yaml @@ -1,36 +1,16 @@ -kind: KubeadmControlPlane -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate metadata: - name: "${CLUSTER_NAME}-control-plane" + name: ${CLUSTER_NAME}-worker-a spec: - version: ${KUBERNETES_VERSION} - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - machineTemplate: - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: PacketMachineTemplate - name: "${CLUSTER_NAME}-control-plane" - kubeadmConfigSpec: - clusterConfiguration: - apiServer: - extraArgs: - cloud-provider: external - controllerManager: - extraArgs: - cloud-provider: external - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-provider: external - provider-id: "equinixmetal://{{ `{{ v1.instance_id }}` }}" - joinConfiguration: - nodeRegistration: - ignorePreflightErrors: - - DirAvailable--etc-kubernetes-manifests - kubeletExtraArgs: - cloud-provider: external - provider-id: "equinixmetal://{{ `{{ v1.instance_id }}` }}" - preKubeadmCommands: + template: + spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + provider-id: equinixmetal://{{ `{{ v1.instance_id }}` }} + preKubeadmCommands: - | sed -ri '/\sswap\s/s/^#?/#/' /etc/fstab swapoff -a @@ -51,10 +31,6 @@ spec: apt-get update -y apt-get remove -y docker docker-engine containerd runc apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq - major_vers=$(lsb_release -r | awk '{ print $2 }' | cut -d. -f1) - if [ "$major_vers" -ge 20 ]; then - apt-get install -y kubetail - fi install -m 0755 -d /etc/apt/keyrings curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg MINOR_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | cut -d. -f1-2 ) @@ -67,86 +43,46 @@ spec: TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//') RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1) apt-get install -y containerd.io kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION} - containerd config default > /etc/containerd/config.toml cat < /etc/crictl.yaml runtime-endpoint: unix:///run/containerd/containerd.sock image-endpoint: unix:///run/containerd/containerd.sock EOF + containerd config default > /etc/containerd/config.toml sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml systemctl restart containerd - if [ -f "/run/kubeadm/kubeadm.yaml" ]; then - ip addr add {{ .controlPlaneEndpoint }} dev lo - fi - postKubeadmCommands: - - | - mkdir -p $HOME/.kube - cp /etc/kubernetes/admin.conf $HOME/.kube/config - echo "source <(kubectl completion bash)" >> $HOME/.bashrc - echo "alias k=kubectl" >> $HOME/.bashrc - echo "complete -o default -F __start_kubectl k" >> $HOME/.bashrc - if [ -f "/run/kubeadm/kubeadm.yaml" ]; then - export KUBECONFIG=/etc/kubernetes/admin.conf - export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/${CPEM_VERSION:=v3.8.0}/deployment.yaml - export SECRET_DATA='cloud-sa.json=''{"apiKey": "{{ .apiKey }}","projectID": "${PROJECT_ID}"}''' - kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}" || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}") || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}") - kubectl apply -f $${CPEM_YAML} || (sleep 1 && kubectl apply -f $${CPEM_YAML}) || (sleep 1 && kubectl apply -f $${CPEM_YAML}) - fi ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: PacketMachineTemplate -metadata: - name: "${CLUSTER_NAME}-control-plane" -spec: - template: - spec: - os: "${NODE_OS:=ubuntu_20_04}" - billingCycle: hourly - machineType: "${CONTROLPLANE_NODE_TYPE}" - sshKeys: - - "${SSH_KEY}" - tags: [] --- apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: - name: "${CLUSTER_NAME}" + name: ${CLUSTER_NAME} spec: clusterNetwork: pods: cidrBlocks: - - ${POD_CIDR:=192.168.0.0/16} + - ${POD_CIDR:=192.168.0.0/16} services: cidrBlocks: - - ${SERVICE_CIDR:=172.26.0.0/16} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: PacketCluster - name: "${CLUSTER_NAME}" + - ${SERVICE_CIDR:=172.26.0.0/16} controlPlaneRef: apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlane - name: "${CLUSTER_NAME}-control-plane" ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: PacketCluster -metadata: - name: "${CLUSTER_NAME}" -spec: - projectID: "${PROJECT_ID}" - metro: "${METRO}" - vipManager: "EMLB" + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: PacketCluster + name: ${CLUSTER_NAME} --- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment metadata: - name: ${CLUSTER_NAME}-worker-a labels: cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} pool: worker-a + name: ${CLUSTER_NAME}-worker-a spec: - replicas: ${WORKER_MACHINE_COUNT} clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} selector: matchLabels: cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} @@ -157,82 +93,146 @@ spec: cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} pool: worker-a spec: - version: ${KUBERNETES_VERSION} - clusterName: ${CLUSTER_NAME} bootstrap: configRef: - name: ${CLUSTER_NAME}-worker-a apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-worker-a + clusterName: ${CLUSTER_NAME} infrastructureRef: - name: ${CLUSTER_NAME}-worker-a apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: PacketMachineTemplate + name: ${CLUSTER_NAME}-worker-a + version: ${KUBERNETES_VERSION} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-provider: external + controllerManager: + extraArgs: + cloud-provider: external + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + provider-id: equinixmetal://{{ `{{ v1.instance_id }}` }} + joinConfiguration: + nodeRegistration: + ignorePreflightErrors: + - DirAvailable--etc-kubernetes-manifests + kubeletExtraArgs: + cloud-provider: external + provider-id: equinixmetal://{{ `{{ v1.instance_id }}` }} + postKubeadmCommands: + - |- + mkdir -p $HOME/.kube + cp /etc/kubernetes/admin.conf $HOME/.kube/config + echo "source <(kubectl completion bash)" >> $HOME/.bashrc + echo "alias k=kubectl" >> $HOME/.bashrc + echo "complete -o default -F __start_kubectl k" >> $HOME/.bashrc + if [ -f "/run/kubeadm/kubeadm.yaml" ]; then + export KUBECONFIG=/etc/kubernetes/admin.conf + export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/${CPEM_VERSION:=v3.8.0}/deployment.yaml + export SECRET_DATA='cloud-sa.json=''{"apiKey": "{{ .apiKey }}","projectID": "${PROJECT_ID}"}''' + kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}" || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}") || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}") + kubectl apply -f $${CPEM_YAML} || (sleep 1 && kubectl apply -f $${CPEM_YAML}) || (sleep 1 && kubectl apply -f $${CPEM_YAML}) + fi + preKubeadmCommands: + - | + sed -ri '/\sswap\s/s/^#?/#/' /etc/fstab + swapoff -a + mount -a + cat < /etc/modules-load.d/containerd.conf + overlay + br_netfilter + EOF + modprobe overlay + modprobe br_netfilter + cat < /etc/sysctl.d/99-kubernetes-cri.conf + net.bridge.bridge-nf-call-iptables = 1 + net.ipv4.ip_forward = 1 + net.bridge.bridge-nf-call-ip6tables = 1 + EOF + sysctl --system + export DEBIAN_FRONTEND=noninteractive + apt-get update -y + apt-get remove -y docker docker-engine containerd runc + apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq + major_vers=$(lsb_release -r | awk '{ print $2 }' | cut -d. -f1) + if [ "$major_vers" -ge 20 ]; then + apt-get install -y kubetail + fi + install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg + MINOR_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | cut -d. -f1-2 ) + curl -fsSL https://pkgs.k8s.io/core:/stable:/$${MINOR_KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + chmod a+r /etc/apt/keyrings/docker.gpg + chmod a+r /etc/apt/keyrings/kubernetes-archive-keyring.gpg + echo "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" > /etc/apt/sources.list.d/docker.list + echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$${MINOR_KUBERNETES_VERSION}/deb/ /" > /etc/apt/sources.list.d/kubernetes.list + apt-get update -y + TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//') + RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1) + apt-get install -y containerd.io kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION} + containerd config default > /etc/containerd/config.toml + cat < /etc/crictl.yaml + runtime-endpoint: unix:///run/containerd/containerd.sock + image-endpoint: unix:///run/containerd/containerd.sock + EOF + sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml + sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml + systemctl restart containerd + if [ -f "/run/kubeadm/kubeadm.yaml" ]; then + ip addr add {{ .controlPlaneEndpoint }} dev lo + fi + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: PacketMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: PacketCluster +metadata: + name: ${CLUSTER_NAME} +spec: + metro: ${METRO} + projectID: ${PROJECT_ID} + vipManager: EMLB --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: PacketMachineTemplate metadata: - name: ${CLUSTER_NAME}-worker-a + name: ${CLUSTER_NAME}-control-plane spec: template: spec: - os: "${NODE_OS:=ubuntu_20_04}" billingCycle: hourly - machineType: "${WORKER_NODE_TYPE}" + machineType: ${CONTROLPLANE_NODE_TYPE} + os: ${NODE_OS:=ubuntu_20_04} sshKeys: - - "${SSH_KEY}" + - ${SSH_KEY} tags: [] --- -kind: KubeadmConfigTemplate -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: PacketMachineTemplate metadata: - name: "${CLUSTER_NAME}-worker-a" + name: ${CLUSTER_NAME}-worker-a spec: template: spec: - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-provider: external - provider-id: "equinixmetal://{{ `{{ v1.instance_id }}` }}" - preKubeadmCommands: - - | - sed -ri '/\sswap\s/s/^#?/#/' /etc/fstab - swapoff -a - mount -a - cat < /etc/modules-load.d/containerd.conf - overlay - br_netfilter - EOF - modprobe overlay - modprobe br_netfilter - cat < /etc/sysctl.d/99-kubernetes-cri.conf - net.bridge.bridge-nf-call-iptables = 1 - net.ipv4.ip_forward = 1 - net.bridge.bridge-nf-call-ip6tables = 1 - EOF - sysctl --system - export DEBIAN_FRONTEND=noninteractive - apt-get update -y - apt-get remove -y docker docker-engine containerd runc - apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq - install -m 0755 -d /etc/apt/keyrings - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg - MINOR_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | cut -d. -f1-2 ) - curl -fsSL https://pkgs.k8s.io/core:/stable:/$${MINOR_KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg - chmod a+r /etc/apt/keyrings/docker.gpg - chmod a+r /etc/apt/keyrings/kubernetes-archive-keyring.gpg - echo "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" > /etc/apt/sources.list.d/docker.list - echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$${MINOR_KUBERNETES_VERSION}/deb/ /" > /etc/apt/sources.list.d/kubernetes.list - apt-get update -y - TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//') - RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1) - apt-get install -y containerd.io kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION} - cat < /etc/crictl.yaml - runtime-endpoint: unix:///run/containerd/containerd.sock - image-endpoint: unix:///run/containerd/containerd.sock - EOF - containerd config default > /etc/containerd/config.toml - sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml - sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml - systemctl restart containerd + billingCycle: hourly + machineType: ${WORKER_NODE_TYPE} + os: ${NODE_OS:=ubuntu_20_04} + sshKeys: + - ${SSH_KEY} + tags: [] diff --git a/templates/experimental-emlb-crs-cni/kustomization.yaml b/templates/experimental-emlb-crs-cni/kustomization.yaml new file mode 100644 index 000000000..61e089ec1 --- /dev/null +++ b/templates/experimental-emlb-crs-cni/kustomization.yaml @@ -0,0 +1,16 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../experimental-emlb + - ../bases/crs-cni.yaml +patches: + - patch: |- + kind: Cluster + apiVersion: cluster.x-k8s.io/v1beta1 + metadata: + name: not-used + labels: + cni: "${CLUSTER_NAME}-crs-cni" + target: + kind: Cluster diff --git a/templates/experimental-emlb/kustomization.yaml b/templates/experimental-emlb/kustomization.yaml new file mode 100644 index 000000000..28fd776fd --- /dev/null +++ b/templates/experimental-emlb/kustomization.yaml @@ -0,0 +1,82 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../cluster-template.yaml +patches: + - patch: |- + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: PacketCluster + metadata: + name: "${CLUSTER_NAME}" + spec: + vipManager: "EMLB" + - patch: |- + kind: KubeadmControlPlane + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + metadata: + name: "${CLUSTER_NAME}-control-plane" + spec: + kubeadmConfigSpec: + preKubeadmCommands: + - | + sed -ri '/\sswap\s/s/^#?/#/' /etc/fstab + swapoff -a + mount -a + cat < /etc/modules-load.d/containerd.conf + overlay + br_netfilter + EOF + modprobe overlay + modprobe br_netfilter + cat < /etc/sysctl.d/99-kubernetes-cri.conf + net.bridge.bridge-nf-call-iptables = 1 + net.ipv4.ip_forward = 1 + net.bridge.bridge-nf-call-ip6tables = 1 + EOF + sysctl --system + export DEBIAN_FRONTEND=noninteractive + apt-get update -y + apt-get remove -y docker docker-engine containerd runc + apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq + major_vers=$(lsb_release -r | awk '{ print $2 }' | cut -d. -f1) + if [ "$major_vers" -ge 20 ]; then + apt-get install -y kubetail + fi + install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg + MINOR_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | cut -d. -f1-2 ) + curl -fsSL https://pkgs.k8s.io/core:/stable:/$${MINOR_KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + chmod a+r /etc/apt/keyrings/docker.gpg + chmod a+r /etc/apt/keyrings/kubernetes-archive-keyring.gpg + echo "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" > /etc/apt/sources.list.d/docker.list + echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$${MINOR_KUBERNETES_VERSION}/deb/ /" > /etc/apt/sources.list.d/kubernetes.list + apt-get update -y + TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//') + RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1) + apt-get install -y containerd.io kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION} + containerd config default > /etc/containerd/config.toml + cat < /etc/crictl.yaml + runtime-endpoint: unix:///run/containerd/containerd.sock + image-endpoint: unix:///run/containerd/containerd.sock + EOF + sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml + sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml + systemctl restart containerd + if [ -f "/run/kubeadm/kubeadm.yaml" ]; then + ip addr add {{ .controlPlaneEndpoint }} dev lo + fi + postKubeadmCommands: + - | + mkdir -p $HOME/.kube + cp /etc/kubernetes/admin.conf $HOME/.kube/config + echo "source <(kubectl completion bash)" >> $HOME/.bashrc + echo "alias k=kubectl" >> $HOME/.bashrc + echo "complete -o default -F __start_kubectl k" >> $HOME/.bashrc + if [ -f "/run/kubeadm/kubeadm.yaml" ]; then + export KUBECONFIG=/etc/kubernetes/admin.conf + export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/${CPEM_VERSION:=v3.8.0}/deployment.yaml + export SECRET_DATA='cloud-sa.json=''{"apiKey": "{{ .apiKey }}","projectID": "${PROJECT_ID}"}''' + kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}" || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}") || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}") + kubectl apply -f $${CPEM_YAML} || (sleep 1 && kubectl apply -f $${CPEM_YAML}) || (sleep 1 && kubectl apply -f $${CPEM_YAML}) + fi diff --git a/test/e2e/capi_e2e_quickstart_test.go b/test/e2e/capi_e2e_quickstart_test.go index 0f9257d1e..40fac5462 100644 --- a/test/e2e/capi_e2e_quickstart_test.go +++ b/test/e2e/capi_e2e_quickstart_test.go @@ -54,4 +54,17 @@ var _ = Describe("[QuickStart] Running the Cluster API E2E QuickStart tests", fu } }) }) + + Context("Running the [EMLB] quickstart spec", func() { + capi_e2e.QuickStartSpec(ctx, func() capi_e2e.QuickStartSpecInput { + return capi_e2e.QuickStartSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + Flavor: ptr.To[string]("emlb"), + } + }) + }) }) diff --git a/test/e2e/config/packet-ci-actions.yaml b/test/e2e/config/packet-ci-actions.yaml index 5a34efaa7..d9012f4fb 100644 --- a/test/e2e/config/packet-ci-actions.yaml +++ b/test/e2e/config/packet-ci-actions.yaml @@ -70,6 +70,7 @@ providers: files: - sourcePath: "${MANIFEST_PATH:=..}/metadata.yaml" - sourcePath: "../data/v1beta1/cluster-template.yaml" + - sourcePath: "../data/v1beta1/cluster-template-emlb.yaml" - sourcePath: "../data/v1beta1/cluster-template-kube-vip.yaml" - sourcePath: "../data/v1beta1/cluster-template-kcp-scale-in.yaml" - sourcePath: "../data/v1beta1/cluster-template-node-drain.yaml" diff --git a/test/e2e/config/packet-ci.yaml b/test/e2e/config/packet-ci.yaml index b03d8b530..765797bc9 100644 --- a/test/e2e/config/packet-ci.yaml +++ b/test/e2e/config/packet-ci.yaml @@ -74,6 +74,7 @@ providers: files: - sourcePath: "../../../metadata.yaml" - sourcePath: "../data/v1beta1/cluster-template.yaml" + - sourcePath: "../data/v1beta1/cluster-template-emlb.yaml" - sourcePath: "../data/v1beta1/cluster-template-kube-vip.yaml" - sourcePath: "../data/v1beta1/cluster-template-kcp-scale-in.yaml" - sourcePath: "../data/v1beta1/cluster-template-node-drain.yaml" @@ -88,6 +89,7 @@ providers: files: - sourcePath: "../../../metadata.yaml" - sourcePath: "../data/v1beta1/cluster-template.yaml" + - sourcePath: "../data/v1beta1/cluster-template-emlb.yaml" - sourcePath: "../data/v1beta1/cluster-template-kube-vip.yaml" - sourcePath: "../data/v1beta1/cluster-template-kcp-scale-in.yaml" - sourcePath: "../data/v1beta1/cluster-template-node-drain.yaml" diff --git a/test/e2e/data/v1beta1/cluster-template-emlb/kustomization.yaml b/test/e2e/data/v1beta1/cluster-template-emlb/kustomization.yaml new file mode 100644 index 000000000..c0a1e6d18 --- /dev/null +++ b/test/e2e/data/v1beta1/cluster-template-emlb/kustomization.yaml @@ -0,0 +1,3 @@ +resources: + - ../../../../../templates/experimental-emlb + - ../bases/kcp-mhc.yaml