diff --git a/guides/flannel.md b/guides/flannel.md index 0561ad4..faafaa1 100644 --- a/guides/flannel.md +++ b/guides/flannel.md @@ -59,40 +59,43 @@ The output should include the Linux flannel DaemonSet as running: ``` NAMESPACE NAME READY STATUS RESTARTS AGE ... -kube-system kube-flannel-ds-54954 1/1 Running 0 1m +kube-flannel kube-flannel-ds-sfqkv 1/1 Running 0 1m ``` ### Add Windows flannel solution -1. Add Windows Flannel and kube-proxy DaemonSets - -Now you can add Windows-compatible versions of Flannel and kube-proxy. In order to ensure that you get a compatible version of kube-proxy, you'll need to substitute the tag of the image. The following example shows usage for Kubernetes v1.27.1, but you should adjust the version for your own deployment. +Now you can add a Windows-compatible version of Flannel. ```bash -curl -L https://raw.githubusercontent.com/kubernetes-sigs/sig-windows-tools/master/hostprocess/flannel/flanneld/flannel-overlay.yml | sed 's/FLANNEL_VERSION/v0.21.5/g' | kubectl apply -f - -curl -L https://raw.githubusercontent.com/kubernetes-sigs/sig-windows-tools/master/hostprocess/flannel/kube-proxy/kube-proxy.yml | sed 's/KUBE_PROXY_VERSION/v1.27.1/g' | kubectl apply -f - +controlPlaneEndpoint=$(kubectl get configmap -n kube-system kube-proxy -o jsonpath="{.data['kubeconfig\.conf']}" | grep server: | sed 's/.*\:\/\///g') +kubernetesServiceHost=$(echo $controlPlaneEndpoint | cut -d ":" -f 1) +kubernetesServicePort=$(echo $controlPlaneEndpoint | cut -d ":" -f 2) +curl -L https://raw.githubusercontent.com/kubernetes-sigs/sig-windows-tools/master/hostprocess/flannel/flanneld/flannel-overlay.yml | sed 's/FLANNEL_VERSION/v0.21.5/g' | sed "s/KUBERNETES_SERVICE_HOST_VALUE/$kubernetesServiceHost/g" | sed "s/KUBERNETES_SERVICE_PORT_VALUE/$kubernetesServicePort/g" | kubectl apply -f - ``` -> **Note** If you are using another version of kubernetes on your Windows node, change v1.27.1 with your own version . -> To find your version of kubernetes run the following command: -> `kubeadm version` +> **Note** If your cluster uses a different service subnet than `10.96.0.0/12` then you need to adjust the environment variable `SERVICE_SUBNET` before applying it. +> To find your service subnet run the following command: +> `kubectl get configmap -n kube-system kubeadm-config -o yaml | grep serviceSubnet` > **Note** If you changed the`$CNIBinPath` or `$CNIConfigPath` optional parameters when running `Install-Containerd.ps1`, -> you will need to use those paths on `flannel-overlay.yml`. Pipe it through +> you will need to use those paths. Pipe it through > `| sed 's/C:\\\\opt\\\\cni\\\\bin//g' | sed 's/C:\\\\etc\\\\cni\\\\net.d//g'` > before feeding it to `kubectl apply -f -`. -> **Note** If you changed the`$CNIBinPath` optional parameter when running `Install-Containerd.ps1`, you will need to -> use that path on `kube-proxy.yml`. Pipe it through `| sed 's/C:\\\\opt\\\\cni\\\\bin//g'` before -> feeding it to `kubectl apply -f -`. - -2. Apply kube-flannel-rbac.yml from sig-windows-tools/kubeadm/flannel -Next you will need to apply the configuration that allows flannel to spawn pods and keep them running: +Next add a Windows-compatible version of kube-proxy. In order to ensure that you get a compatible version of kube-proxy, you'll need to substitute the tag of the image. The following example shows usage for Kubernetes v1.27.1, but you should adjust the version for your own deployment. ```bash -kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/sig-windows-tools/master/hostprocess/flannel/flanneld/kube-flannel-rbac.yml +curl -L https://raw.githubusercontent.com/kubernetes-sigs/sig-windows-tools/master/hostprocess/flannel/kube-proxy/kube-proxy.yml | sed 's/KUBE_PROXY_VERSION/v1.27.1/g' | kubectl apply -f - ``` +> **Note** If you are using another version of kubernetes on your Windows node, change v1.27.1 with your own version . +> To find your version of kubernetes run the following command: +> `kubeadm version` + +> **Note** If you changed the`$CNIBinPath` optional parameter when running `Install-Containerd.ps1`, you will need to +> use that path. Pipe it through `| sed 's/C:\\\\opt\\\\cni\\\\bin//g'` before +> feeding it to `kubectl apply -f -`. + ## Verifying your installation for Flannel You should now be able to view the Windows node in your cluster by running: @@ -101,10 +104,10 @@ You should now be able to view the Windows node in your cluster by running: kubectl get nodes -o wide ``` -If your new node is in the `NotReady` state it is likely because the flannel image is still downloading. You can check the progress as before by checking on the flannel pods in the `kube-system` namespace: +If your new node is in the `NotReady` state it is likely because the flannel image is still downloading. You can check the progress as before by checking on the flannel pods in the `kube-flannel` namespace: ```shell -kubectl -n kube-system get pods -l app=flannel +kubectl -n kube-flannel get pods -l app=flannel ``` Once the flannel Pod is running, your node should enter the `Ready` state and then be available to handle workloads. diff --git a/hostprocess/flannel/flanneld/flannel-overlay.yml b/hostprocess/flannel/flanneld/flannel-overlay.yml index 61b6555..5dae117 100644 --- a/hostprocess/flannel/flanneld/flannel-overlay.yml +++ b/hostprocess/flannel/flanneld/flannel-overlay.yml @@ -3,7 +3,7 @@ kind: ConfigMap apiVersion: v1 metadata: name: kube-flannel-windows-cfg - namespace: kube-system + namespace: kube-flannel labels: tier: node app: flannel @@ -59,7 +59,7 @@ metadata: labels: tier: node app: flannel - namespace: kube-system + namespace: kube-flannel spec: selector: matchLabels: @@ -106,15 +106,23 @@ spec: mountPath: /mounts/kube-flannel/ - name: flannel-windows-cfg mountPath: /mounts/kube-flannel-windows/ - - name: kubeadm-config - mountPath: /mounts/kubeadm-config/ - - name: kube-proxy - mountPath: /flannel-config-file env: - name: CNI_BIN_PATH value: C:\\opt\\cni\\bin - name: CNI_CONFIG_PATH value: C:\\etc\\cni\\net.d + - name: SERVICE_SUBNET + value: 10.96.0.0/12 + # As of now with the currently used flannel version (last checked with v0.21.5) we need to overwrite KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT + # in order to be able to reach the kubernetes api server. Under windows it's currently not possible to reach it over the service created by kubernetes + # For more context and details check the corresponding PR: https://github.com/kubernetes-sigs/sig-windows-tools/pull/314 + # Especially the comments in this review: https://github.com/kubernetes-sigs/sig-windows-tools/pull/314#discussion_r1238815189 + # There is also a follow up issue on the flannel side: https://github.com/flannel-io/flannel/issues/1772 + # Once this issue is solved we should be able to remove the custom host and port to the kubernetes api server + - name: KUBERNETES_SERVICE_HOST + value: KUBERNETES_SERVICE_HOST_VALUE + - name: KUBERNETES_SERVICE_PORT + value: "KUBERNETES_SERVICE_PORT_VALUE" - name: POD_NAME valueFrom: fieldRef: @@ -126,15 +134,9 @@ spec: apiVersion: v1 fieldPath: metadata.namespace volumes: - - name: kube-proxy - configMap: - name: kube-proxy - name: flannel-cfg configMap: name: kube-flannel-cfg - name: flannel-windows-cfg configMap: name: kube-flannel-windows-cfg - - name: kubeadm-config - configMap: - name: kubeadm-config diff --git a/hostprocess/flannel/flanneld/kube-flannel-rbac.yml b/hostprocess/flannel/flanneld/kube-flannel-rbac.yml deleted file mode 100644 index 65e7b35..0000000 --- a/hostprocess/flannel/flanneld/kube-flannel-rbac.yml +++ /dev/null @@ -1,57 +0,0 @@ ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: flannel - namespace: kube-system ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: flannel-windows -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: flannel -subjects: - - kind: ServiceAccount - name: flannel - namespace: kube-system ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: kube-flannel-cfg - namespace: kube-system - labels: - tier: node - app: flannel -data: - cni-conf.json: | - { - "name": "cbr0", - "plugins": [ - { - "type": "flannel", - "delegate": { - "hairpinMode": true, - "isDefaultGateway": true - } - }, - { - "type": "portmap", - "capabilities": { - "portMappings": true - } - } - ] - } - net-conf.json: | - { - "Network": "10.244.0.0/16", - "Backend": { - "Type": "vxlan", - "VNI" : 4096, - "Port": 4789 - } - } diff --git a/hostprocess/flannel/flanneld/start.ps1 b/hostprocess/flannel/flanneld/start.ps1 index fcbde5a..21f1efe 100644 --- a/hostprocess/flannel/flanneld/start.ps1 +++ b/hostprocess/flannel/flanneld/start.ps1 @@ -15,8 +15,8 @@ cp -force $env:CONTAINER_SANDBOX_MOUNT_POINT/mounts/kube-flannel/net-conf.json # get info Write-Host "update cni config" $cniJson = get-content $env:CONTAINER_SANDBOX_MOUNT_POINT/mounts/kube-flannel-windows/cni-conf-containerd.json | ConvertFrom-Json -$serviceSubnet = get-content $env:CONTAINER_SANDBOX_MOUNT_POINT/mounts/kubeadm-config/ClusterConfiguration | ForEach-Object -Process {if($_.Contains("serviceSubnet:")) {$_.Trim().Split()[1]}} -$podSubnet = get-content $env:CONTAINER_SANDBOX_MOUNT_POINT/mounts/kubeadm-config/ClusterConfiguration | ForEach-Object -Process {if($_.Contains("podSubnet:")) {$_.Trim().Split()[1]}} +$serviceSubnet = $env:SERVICE_SUBNET +$podSubnet = (get-content $env:CONTAINER_SANDBOX_MOUNT_POINT/mounts/kube-flannel/net-conf.json | ConvertFrom-Json).Network $na = Get-NetRoute | Where { $_.DestinationPrefix -eq '0.0.0.0/0' } | Select-Object -Property ifIndex $managementIP = (Get-NetIPAddress -ifIndex $na[0].ifIndex -AddressFamily IPv4).IPAddress @@ -32,13 +32,9 @@ Set-Content -Path $env:CNI_CONFIG_PATH/10-flannel.conf ($cniJson | ConvertTo-Jso Write-Host "add route" route /p add 169.254.169.254 mask 255.255.255.255 0.0.0.0 -write-host "copy sa info (should be able to do this with a change to go client" -mkdir -force $env:CONTAINER_SANDBOX_MOUNT_POINT/flannel-config-file/var/run/secrets/kubernetes.io/serviceaccount/ -cp -force $env:CONTAINER_SANDBOX_MOUNT_POINT/var/run/secrets/kubernetes.io/serviceaccount/* $env:CONTAINER_SANDBOX_MOUNT_POINT/flannel-config-file/var/run/secrets/kubernetes.io/serviceaccount/ - Write-Host "envs" write-host $env:POD_NAME write-host $env:POD_NAMESPACE Write-Host "Starting flannel" -& $env:CONTAINER_SANDBOX_MOUNT_POINT/flannel/flanneld.exe --kube-subnet-mgr --kubeconfig-file $env:CONTAINER_SANDBOX_MOUNT_POINT/flannel-config-file/kubeconfig.conf --iface $managementIP +& $env:CONTAINER_SANDBOX_MOUNT_POINT/flannel/flanneld.exe --kube-subnet-mgr --iface $managementIP