diff --git a/.ci/version-lock b/.ci/version-lock index bb839e1ac..9bcf6877d 100644 --- a/.ci/version-lock +++ b/.ci/version-lock @@ -1,7 +1,7 @@ # Core Version -argocd_version=4.10.3 -sealed_secrets_version=2.5.2 -kubevirt_version=v0.55.0-rc.0 +argocd_version=4.10.5 +sealed_secrets_version=2.6.0 +kubevirt_version=v0.56.0-rc.0 coredns_version=1.9.3 # Utils Version @@ -12,11 +12,11 @@ etcdctl_version=v3.5.4 # Apps local_path_provisioner_version=v0.0.22 -kube_prometheus_stack_version=39.2.0 +kube_prometheus_stack_version=39.5.0 # cfctl.yaml -k0s_version=1.24.2+k0s.0 -metallb_version=3.0.12 +k0s_version=1.24.3+k0s.0 +metallb_version=4.0.1 traefik_version=10.24.0 cert_manager_version=v1.9.1 csi_driver_nfs_version=v4.1.0 diff --git a/argo.example/monitoring/apps/prometheus-crd-app.yml b/argo.example/monitoring/apps/prometheus-crd-app.yml index 6c70a24d9..6887d0e76 100644 --- a/argo.example/monitoring/apps/prometheus-crd-app.yml +++ b/argo.example/monitoring/apps/prometheus-crd-app.yml @@ -10,7 +10,7 @@ spec: source: repoURL: https://github.com/prometheus-community/helm-charts.git path: charts/kube-prometheus-stack/crds/ - targetRevision: kube-prometheus-stack-39.2.0 + targetRevision: kube-prometheus-stack-39.5.0 directory: recurse: true diff --git a/cfctl.yaml.example b/cfctl.yaml.example index e056f9175..d2e614410 100644 --- a/cfctl.yaml.example +++ b/cfctl.yaml.example @@ -25,7 +25,7 @@ spec: - sh -c 'if [ "$(getenforce)" != "Permissive" ] && [ "$(getenforce)" != "Disabled" ]; then setenforce 0; fi' k0s: - version: '1.24.2+k0s.0' + version: '1.24.3+k0s.0' dynamicConfig: false config: apiVersion: k0s.k0sproject.io/v1beta1 @@ -85,24 +85,8 @@ spec: charts: - name: metallb chartname: bitnami/metallb - version: '3.0.12' + version: '4.0.1' namespace: metallb - values: | - configInline: - peers: - - peer-address: 192.168.0.1 - peer-asn: 65000 - my-asn: 65001 - source-address: 192.168.0.2 - node-selectors: - - match-labels: - kubernetes.io/hostname: mn1.example.com - - address-pools: - - name: main-pool - protocol: bgp - addresses: - - 192.168.1.100/32 - name: traefik chartname: traefik/traefik diff --git a/core.example/argo-cd/install.sh b/core.example/argo-cd/install.sh index cc9708b24..cc00e87f1 100755 --- a/core.example/argo-cd/install.sh +++ b/core.example/argo-cd/install.sh @@ -8,7 +8,7 @@ helm repo update helm upgrade --install \ -n argocd \ -f "${BASEDIR}/values.yaml" \ - --version 4.10.3 \ + --version 4.10.5 \ argocd \ argo/argo-cd \ --create-namespace diff --git a/core.example/kubevirt/base/kustomization.yaml b/core.example/kubevirt/base/kustomization.yaml index 9683073ec..58a907251 100644 --- a/core.example/kubevirt/base/kustomization.yaml +++ b/core.example/kubevirt/base/kustomization.yaml @@ -1,3 +1,3 @@ resources: - - https://github.com/kubevirt/kubevirt/releases/download/v0.55.0-rc.0/kubevirt-operator.yaml - - https://github.com/kubevirt/kubevirt/releases/download/v0.55.0-rc.0/kubevirt-cr.yaml + - https://github.com/kubevirt/kubevirt/releases/download/v0.56.0-rc.0/kubevirt-operator.yaml + - https://github.com/kubevirt/kubevirt/releases/download/v0.56.0-rc.0/kubevirt-cr.yaml diff --git a/core.example/metallb/address-pools.yaml b/core.example/metallb/address-pools.yaml new file mode 100644 index 000000000..6dade2f10 --- /dev/null +++ b/core.example/metallb/address-pools.yaml @@ -0,0 +1,8 @@ +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: main-pool + namespace: metallb +spec: + addresses: + - 192.168.1.100/32 diff --git a/core.example/metallb/advertisements.yaml b/core.example/metallb/advertisements.yaml new file mode 100644 index 000000000..605b8d459 --- /dev/null +++ b/core.example/metallb/advertisements.yaml @@ -0,0 +1,8 @@ +apiVersion: metallb.io/v1beta1 +kind: BGPAdvertisement +metadata: + name: bgp-advertisement + namespace: metallb +spec: + ipAddressPools: + - main-pool diff --git a/core.example/metallb/peers.yaml b/core.example/metallb/peers.yaml new file mode 100644 index 000000000..762f77ff5 --- /dev/null +++ b/core.example/metallb/peers.yaml @@ -0,0 +1,13 @@ +apiVersion: metallb.io/v1beta2 +kind: BGPPeer +metadata: + name: main-router + namespace: metallb +spec: + myASN: 65001 # MetalLB Speaker ASN (Autonomous System Number) + nodeSelectors: + - matchLabels: + kubernetes.io/hostname: mn1.example.com # The speaker node, which is the entrypoint. + peerASN: 65000 # The router ASN + peerAddress: 192.168.0.1 # The router address + sourceAddress: 192.168.0.2 # The speaker node IP, which is used to identity the network interface of the node. diff --git a/core.example/sealed-secrets/install.sh b/core.example/sealed-secrets/install.sh index 579b92eae..e03f86280 100755 --- a/core.example/sealed-secrets/install.sh +++ b/core.example/sealed-secrets/install.sh @@ -5,7 +5,7 @@ helm repo update helm upgrade --install \ -n sealed-secrets \ - --version 2.5.2 \ + --version 2.6.0 \ sealed-secrets \ sealed-secrets/sealed-secrets \ --create-namespace diff --git a/helm-subcharts/kube-prometheus-stack/Chart.yaml b/helm-subcharts/kube-prometheus-stack/Chart.yaml index 8eed39b3a..85cf3419c 100644 --- a/helm-subcharts/kube-prometheus-stack/Chart.yaml +++ b/helm-subcharts/kube-prometheus-stack/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kube-prometheus-stack-subchart description: Kube Prometheus Stack subchart type: application -version: 39.2.0 +version: 39.5.0 appVersion: '36.2.1' dependencies: diff --git a/web/docs/getting-started/03-k0s-configuration.md b/web/docs/getting-started/03-k0s-configuration.md index 3c86e288e..290fa46d8 100644 --- a/web/docs/getting-started/03-k0s-configuration.md +++ b/web/docs/getting-started/03-k0s-configuration.md @@ -44,7 +44,7 @@ After you set the `hosts` field, you must configure the k0s architecture by edit ```yaml title="cfctl.yaml > spec > k0s" k0s: - version: '1.24.2+k0s.0' + version: '1.24.3+k0s.0' dynamicConfig: false config: apiVersion: k0s.k0sproject.io/v1beta1 @@ -98,79 +98,9 @@ If you wish to use a HA setup, please follow [this guide](/docs/guides/maintenan After setting up k0s, you can change the `extensions` field. This field can be changed at any time. You can add or change extensions. However, removing an extension is permanent. -## Configuring MetalLB - -Start with `metallb`. MetalLB is a L2/L3 load balancer designed for bare metal Kubernetes clusters. It exposes the kubernetes `Services` to the external network. It uses either L2 (ARP) or BGP to advertise routes. The network indicated by `metallb` must be outside the network when using BGP. Otherwise, when using L2 (ARP), the network must be the same as your private network. For multi-zone clusters, you MUST use BGP. - -
- -![metallb_concepts](02-k0s-configuration.assets/metallb_concepts.png#white-bg) - -
- -:::note - -MetalLB 0.13.0 will allow you to create "zoned" L2 announcements, which means you can make ARP calls by zone. - -More precisely, this means that you can allow `192.168.0.100` in the network `192.168.0.0/24` in one zone, and `172.24.0.100` in the network `172.24.0.0/18` in an another zone, which means that you wouldn't need BGP anymore. - -However, MetalLB 0.13.0 is not yet available at the time of writing, and BGP is a sure-fire. - -::: - -Your router must be capable of using BGP. If not, you should use an appliance with BGP capabilities (like OPNsense, OpenWRT, vyOS, or Linux with BIRD). - -```yaml title="cfctl.yaml > spec > k0s > spec > extensions > helm > charts[]" -- name: metallb - chartname: bitnami/metallb - version: '3.0.12' - namespace: metallb - values: | - - configInline: - peers: - - peer-address: 192.168.0.1 - peer-asn: 65000 - my-asn: 65001 - source-address: 192.168.0.2 - node-selectors: - - match-labels: - kubernetes.io/hostname: mn1.at1.example.com - - peer-address: 10.10.2.1 - peer-asn: 65002 - my-asn: 65001 - source-address: 10.10.2.2 - node-selectors: - - match-labels: - kubernetes.io/hostname: mn1.ch1.example.com - - address-pools: - - name: main-pool - protocol: bgp - addresses: - - 192.168.1.100/32 -``` - -[Use L2 if you have only one zone](https://metallb.universe.tf/configuration/#layer-2-configuration). - -```yaml title="cfctl.yaml > spec > k0s > spec > extensions > helm > charts[]" -- name: metallb - chartname: bitnami/metallb - version: '3.0.12' - namespace: metallb - values: | - - configInline: - address-pools: - - name: main-pool - protocol: layer2 - addresses: - - 192.168.1.100/32 -``` - ## Configuring Traefik -After configuring the Load Balancer, you should configure Traefik, the main Ingress and L7 load balancer. +You should configure Traefik, which is the main Ingress and L7 load balancer. ```yaml title="cfctl.yaml > spec > k0s > spec > extensions > helm > charts[]" - name: traefik @@ -272,11 +202,13 @@ After configuring the Load Balancer, you should configure Traefik, the main Ingr fsGroup: 65532 ``` -Look for `loadBalancerIP` and use the IPs from the MetalLB. +Since we are using MetalLB, we select our `IPAddressPool` by using the `metallb.universe.tf/address-pool` annotation. In, the next chapter will deploy the `IPAddressPool`. For now, let's assume we only need one `IPAddressPool` which is `main-pool`. + +Look for `loadBalancerIP`, the value of that field we correspond to a IP address included in the `IPAddressPool`. **This IP address will be exposed to the external network.** -Add or remove ports. Since Traefik will be used as the main Ingress, these ports will be exposed to the external network. +After that, you can add or remove ports. Since Traefik will be used as the main Ingress, these ports will be exposed to the external network. -The IngressClass is `traefik`. If you don't want to use Traefik, feel free to add another extension. +The IngressClass is `traefik`. If you don't want to use Traefik as the main Ingress, feel free to add an another extension. We use Traefik because it can do a lot of complex route operations while still being able to do basic HTTP routing. @@ -315,6 +247,7 @@ alias kubectx="kubectl config current-context" Congratulation, you have deployed your Kubernetes cluster! However, it's still missing a few core features: +- MetalLB advertisements, for Load Balancing - CoreDNS, which is the internal DNS for Kubernetes - KubeVirt, to deploy VM workloads - Multus CNI, to support multiple network interfaces diff --git a/web/docs/getting-started/04-core-apps-deployment.md b/web/docs/getting-started/04-core-apps-deployment.md index d58b351a8..4849b548f 100644 --- a/web/docs/getting-started/04-core-apps-deployment.md +++ b/web/docs/getting-started/04-core-apps-deployment.md @@ -2,6 +2,7 @@ We will deploy: +- MetalLB advertisements, for Load Balancing - CoreDNS, the internal DNS for Kubernetes - Sealed Secrets, secret management optimized for GitOps - Cert-manager issuers, to generate your SSL certificates and enable, for free, TLS configuration. @@ -9,6 +10,94 @@ We will deploy: - Multus CNI, to support multiple network interfaces - KubeVirt, to deploy VM workloads +## Configuring MetalLB + +MetalLB is a L2/L3 load balancer designed for bare metal Kubernetes clusters. It exposes the kubernetes `Services` to the external network. It uses either L2 (ARP) or BGP to advertise routes. It is possible to make "zoned" advertisements with L2, but we heavily recommend to use BGP for multi-zone clusters. + +
+ +![metallb_concepts](02-k0s-configuration.assets/metallb_concepts.png#white-bg) + +
+ +### Multi-zone (BGP) + +Your router must be capable of using BGP. If not, you should use an appliance with BGP capabilities (like OPNsense, OpenWRT, vyOS, or Linux with BIRD) which act like a router. + +Let's start configuring the main `IPAddressPool`: + +```yaml title="core/metallb/address-pools.yaml" +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: main-pool + namespace: metallb +spec: + addresses: + - 192.168.1.100/32 +``` + +The indicated IP address will be allocated to the `LoadBalancer` Kubernetes Services, which is Traefik. + +We should now advertise the IP address by configuring a `BGPAdvertisement` and its peers: + +```yaml title="core/metallb/peers.yaml" +apiVersion: metallb.io/v1beta2 +kind: BGPPeer +metadata: + name: main-router + namespace: metallb +spec: + myASN: 65001 # MetalLB Speaker ASN (Autonomous System Number) + peerASN: 65000 # The router ASN + peerAddress: 192.168.0.1 # The router address +``` + +```yaml title="core/metallb/advertisements.yaml" +apiVersion: metallb.io/v1beta1 +kind: BGPAdvertisement +metadata: + name: bgp-advertisement + namespace: metallb +spec: + ipAddressPools: + - main-pool +``` + +With this configuration, the MetalLB speakers on all the nodes will advertise the IP address `192.168.1.100/32` to the router, which is at `192.168.0.1`. By receiving the advertisement, the router will create a BGP route `192.168.1.100/32 via `. + +### Single zone (L2/ARP) + +Let's start configuring the main `IPAddressPool`: + +```yaml title="core/metallb/address-pools.yaml" +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: main-pool + namespace: metallb +spec: + addresses: + - 192.168.1.100/32 +``` + +The indicated IP address will be allocated to the `LoadBalancer` Kubernetes Services, which is Traefik. + +We should now advertise the IP address by configuring a `L2Advertisement`: + +```yaml title="core/metallb/advertisements.yaml" +apiVersion: metallb.io/v1beta1 +kind: L2Advertisement +metadata: + name: l2-advertisement + namespace: metallb +spec: + ipAddressPools: + - main-pool +``` + +That's all! The MetalLB speakers on all the nodes will advertise the IP address `192.168.1.100/32` to the router via ARP. By receiving the advertisement, the router will create a BGP route `192.168.1.100/32 via `. + ## CoreDNS configuration The CoreDNS given by k0s does not meet our needs, so we added `--disable-components coredns` in the `installFlags` of `cfctl.yaml`. We are going to deploy our own. diff --git a/web/docs/getting-started/05-argo-apps-deployment.md b/web/docs/getting-started/05-argo-apps-deployment.md index 582a5ee88..8283fe4c9 100644 --- a/web/docs/getting-started/05-argo-apps-deployment.md +++ b/web/docs/getting-started/05-argo-apps-deployment.md @@ -257,7 +257,7 @@ spec: chart: kube-prometheus-stack repoURL: https://github.com/prometheus-community/helm-charts.git path: charts/kube-prometheus-stack/crds/ - targetRevision: kube-prometheus-stack-39.2.0 + targetRevision: kube-prometheus-stack-39.5.0 directory: recurse: true @@ -292,12 +292,12 @@ apiVersion: v2 name: kube-prometheus-stack-subchart description: Kube Prometheus Stack subchart type: application -version: 39.2.0 +version: 39.5.0 appVersion: '0.1.2' dependencies: - name: kube-prometheus-stack - version: 39.2.0 + version: 39.5.0 repository: https://prometheus-community.github.io/helm-charts ```