feature: support infiniband with ib-sriov and ipoib cni (#2815)

* suport ib-sriov cni Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io> * fix Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io> * fix Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io> * fix Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io> * fix Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io> * fix Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io> * d Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io> * ipoib Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io> * ipoib Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io> * ipoib Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io> * ipoib Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io> * ipoib Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io> * ipoib Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io> * ipoib Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io> * ipoib Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io> * ipoib Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io> * ipoib Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io> * ipoib Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io> * d Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io> * d Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io> * d Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io> * d Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io> * d Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io> * d Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io> * d Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io> * d Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io> * d Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io> * d Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io> * d Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io> * d Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io> * d Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io> * d Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io> * d Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io> * d Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io> * d Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io> --------- Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io>
spidernet-io · Dec 14, 2023 · 00156ed · 00156ed
1 parent 72e615f
commit 00156ed
Show file tree

Hide file tree

Showing 33 changed files with 1,490 additions and 445 deletions.
diff --git a/.github/workflows/build-image-plugins.yaml b/.github/workflows/build-image-plugins.yaml
@@ -23,17 +23,25 @@ on:
         required: true
         default: v0.8.0
       cni_version:
-        description: 'the version of cni-plugins, default to latest tag'
+        description: 'the version of cni-plugins, leave empty for latest release'
         required: false
         default: ""
       ovs_version:
-        description: 'the version of ovs-cni plugin, default to latest tag'
+        description: 'the version of ovs-cni plugin, leave empty for latest release'
         required: false
         default: ""
       rdma_version:
-        description: 'the version of rdma-cni plugin, default to main branch'
+        description: 'the version of rdma-cni plugin, leave empty for latest main code'
         required: false
-        default: "latest"
+        default: ""
+      ibsriov_version:
+        description: 'the version of ib-sriov plugin, leave empty for latest release'
+        required: false
+        default: ""
+      ipoib_version:
+        description: 'the version of ipoib plugin, leave empty for latest release'
+        required: false
+        default: ""
   push:
     branches:
       - main
@@ -72,71 +80,75 @@ jobs:
         id: get_event_version
         continue-on-error: false
         run: |
-          cni_latest_version=$(curl  --retry 10  -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" https://api.github.com/repos/containernetworking/plugins/releases/latest | jq '.tag_name' | tr -d '"')
-          if [ -z "${cni_latest_version}" ] ; then
-            echo "unable to get cni version" && exit 1
-          fi
-          ovs_latest_version=$(curl  --retry 10  -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" https://api.github.com/repos/k8snetworkplumbingwg/ovs-cni/releases/latest | jq '.tag_name' | tr -d '"')
-          if [ -z "${ovs_latest_version}" ] ; then
-            echo "unable to get ovs version" && exit 1
+          if [ -n "${{ github.event.inputs.cni_version }}" ]; then
+              cni_version=${{ github.event.inputs.cni_version }}
+          else
+              cni_version=$(curl  --retry 10  -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" https://api.github.com/repos/containernetworking/plugins/releases/latest | jq '.tag_name' | tr -d '"')
+              if [ -z "${cni_version}" ] ; then
+                echo "unable to get cni version" && exit 1
+              fi
           fi
 
-          rdma_latest_version=$(curl  --retry 10  -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" https://api.github.com/repos/k8snetworkplumbingwg/rdma-cni/releases/latest | jq '.tag_name' | tr -d '"')
-          if [ -z "${rdma_latest_version}" ] ; then
-            echo "unable to get rdma version" && exit 1
+          if [ -n "${{ github.event.inputs.ovs_version }}" ]; then
+              ovs_version=${{ github.event.inputs.ovs_version }}
+          else
+              ovs_version=$(curl  --retry 10  -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" https://api.github.com/repos/k8snetworkplumbingwg/ovs-cni/releases/latest | jq '.tag_name' | tr -d '"')
+              if [ -z "${ovs_version}" ] ; then
+                echo "unable to get ovs version" && exit 1
+              fi
           fi
 
-          if ${{ github.event_name == 'workflow_dispatch' }}; then
-            echo "trigger by workflow_dispatch"
-            cni_version=${{ github.event.inputs.cni_version }}
-            if [ -z "${cni_version}" ]; then
-               cni_version=${cni_latest_version}
-            fi
-
-            ovs_version=${{ github.event.inputs.ovs_version }}
-            if [ -z "${ovs_version}" ]; then
-               ovs_version=${ovs_latest_version}
-            fi
-
-            rdma_version=${{ github.event.inputs.rdma_version }}
-            if [ -z "${rdma_version}" ]; then
-               rdma_version=${rdma_latest_version}
-            elif [ "${rdma_version}" = "latest" ] ; then
+          if [ -n "${{ github.event.inputs.rdma_version }}" ]; then
+              rdma_version=${{ github.event.inputs.rdma_version }}
+          else
                # rdma don't release any more, we use main branch to build.
                git clone https://github.com/k8snetworkplumbingwg/rdma-cni.git
                cd rdma-cni && rdma_version=$(git show -s --format='format:%H') && cd ..
-            fi
-
-            ref=${{ github.event.inputs.ref }}
-            tag=${{ github.event.inputs.tag }}
-            echo "cni version: ${cni_version}"
-            echo "ovs version: ${ovs_version}"
-            echo "rdma version: ${rdma_version}"
-            echo "plugins tag: ${tag}"
-
-            echo "event_ref=${ref}" >> $GITHUB_OUTPUT
-            echo "event_tag=${tag}" >> $GITHUB_OUTPUT
-            echo "event_cni_version=${cni_version}" >> $GITHUB_OUTPUT
-            echo "event_ovs_version=${ovs_version}" >> $GITHUB_OUTPUT
-            echo "event_rdma_version=${rdma_version}" >> $GITHUB_OUTPUT
+          fi
+
+          if [ -n "${{ github.event.inputs.ibsriov_version }}" ]; then
+              ibsriov_version="${{ github.event.inputs.ibsriov_version }}"
+          else
+              ibsriov_version=$(curl  --retry 10  -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" https://api.github.com/repos/k8snetworkplumbingwg/ib-sriov-cni/releases/latest | jq '.tag_name' | tr -d '"')
+              if [ -z "${ibsriov_version}" ] ; then
+                echo "unable to get ibsriov version" && exit 1
+              fi
+          fi
+
+          if [ -n "${{ github.event.inputs.ipoib_version }}" ]; then
+              ipoib_version="${{ github.event.inputs.ipoib_version }}"
+          else
+              ipoib_version=$(curl  --retry 10  -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" https://api.github.com/repos/Mellanox/ipoib-cni/releases/latest | jq '.tag_name' | tr -d '"')
+              if [ -z "${ipoib_version}" ] ; then
+                echo "unable to get ipoib version" && exit 1
+              fi
+          fi
+
+          if ${{ github.event_name == 'workflow_dispatch' }}; then
+              ref=${{ github.event.inputs.ref }}
+              tag=${{ github.event.inputs.tag }}
           elif ${{ github.event_name == 'push' }} ; then
-            echo "trigger by push"
-            git clone https://github.com/k8snetworkplumbingwg/rdma-cni.git
-            cd rdma-cni && rdma_latest_version=$(git show -s --format='format:%H') && cd ..
-
-            echo "cni version: ${cni_latest_version}"
-            echo "ovs version: ${ovs_latest_version}"
-            echo "rdma version: ${rdma_latest_version}"
-            echo "event_ref=${{ github.ref }}" >> $GITHUB_OUTPUT
-            echo "event_tag=${{ github.sha }}" >> $GITHUB_OUTPUT
-            echo "event_cni_version=${cni_latest_version}" >> $GITHUB_OUTPUT
-            echo "event_ovs_version=${ovs_latest_version}" >> $GITHUB_OUTPUT
-            echo "event_rdma_version=${rdma_latest_version}" >> $GITHUB_OUTPUT
+              echo "event_ref=${{ github.ref }}" >> $GITHUB_OUTPUT
+              echo "event_tag=${{ github.sha }}" >> $GITHUB_OUTPUT
           else
-            echo "unexpected event"
-            exit 1
+              echo "unexpected event"
+              exit 1
           fi
 
+          echo "event_ref=${ref}" >> $GITHUB_OUTPUT
+          echo "event_tag=${tag}" >> $GITHUB_OUTPUT
+          echo "image tag: ${tag}"
+          echo "cni version: ${cni_version}"
+          echo "ovs version: ${ovs_version}"
+          echo "rdma version: ${rdma_version}"
+          echo "ibsriov version : ${ibsriov_version}"
+          echo "ipoib version : ${ipoib_version}"
+          echo "event_cni_version=${cni_version}" >> $GITHUB_OUTPUT
+          echo "event_ovs_version=${ovs_version}" >> $GITHUB_OUTPUT
+          echo "event_rdma_version=${rdma_version}" >> $GITHUB_OUTPUT
+          echo "event_ibsriov_version=${ibsriov_version}" >> $GITHUB_OUTPUT
+          echo "event_ipoib_version=${ipoib_version}" >> $GITHUB_OUTPUT
+
       - name: Checkout Source Code
         uses: actions/checkout@v4
         with:
@@ -193,6 +205,8 @@ jobs:
             CNI_VERSION=${{ steps.get_event_version.outputs.event_cni_version }}
             OVS_VERSION=${{ steps.get_event_version.outputs.event_ovs_version }}
             RDMA_VERSION=${{ steps.get_event_version.outputs.event_rdma_version }}
+            IB_SRIOV_VERSION=${{ steps.get_event_version.outputs.event_ibsriov_version }}
+            IPOIB_VERSION=${{ steps.get_event_version.outputs.event_ipoib_version }}
 
       - name: Image Release Digest
         if: ${{ steps.tag-in-repositories.outputs.exists == 'false' }}

diff --git a/README-zh_CN.md b/README-zh_CN.md
@@ -60,7 +60,7 @@ underlay CNI 主要指 macvlan、ipvlan、SR-IOV 等能够直接访问宿主机
 
 * eBPF 增强
 
-    kube-proxy replacement 技术极大加速了访问 service 场景，同节点上的 socket 短路技术加速了本地 Pod 的通信效率。相比 kube proxy 解析方式，[网络延时有最大 25% 的改善，网络吞吐有 50% 的提高]((./docs/concepts/io-performance-zh_CN.md))。
+    kube-proxy replacement 技术极大加速了访问 service 场景，同节点上的 socket 短路技术加速了本地 Pod 的通信效率。相比 kube proxy 解析方式，[网络延时有最大 25% 的改善，网络吞吐有 50% 的提高](./docs/concepts/io-performance-zh_CN.md) 。
 
 * RDMA
 

diff --git a/charts/spiderpool/README.md b/charts/spiderpool/README.md
@@ -214,13 +214,15 @@ helm install spiderpool spiderpool/spiderpool --wait --namespace kube-system \
 | Name                             | Description                                                | Value                                        |
 | -------------------------------- | ---------------------------------------------------------- | -------------------------------------------- |
 | `plugins.installCNI`             | install all cni plugins to each node                       | `false`                                      |
-| `plugins.installRdmaCNI`         | install rdma cni used to isolate rdma device for sriov cni | `false`                                      |
-| `plugins.installOvsCNI`          | install ovs cni to each node                               | `false`                                      |
+| `plugins.installRdmaCNI`         | install rdma cni used to isolate rdma device for sriov cni | `true`                                       |
+| `plugins.installOvsCNI`          | install ovs cni to each node                               | `true`                                       |
+| `plugins.installibSriovCNI`      | install ib-sriov cni to each node                          | `true`                                       |
+| `plugins.installIpoibCNI`        | install ipoib cni to each node                             | `true`                                       |
 | `plugins.image.registry`         | the image registry of plugins                              | `ghcr.io`                                    |
 | `plugins.image.repository`       | the image repository of plugins                            | `spidernet-io/spiderpool/spiderpool-plugins` |
 | `plugins.image.pullPolicy`       | the image pullPolicy of plugins                            | `IfNotPresent`                               |
 | `plugins.image.digest`           | the image digest of plugins                                | `""`                                         |
-| `plugins.image.tag`              | the image tag of plugins                                   | `v0.8.0`                                     |
+| `plugins.image.tag`              | the image tag of plugins                                   | `v0.9.2`                                     |
 | `plugins.image.imagePullSecrets` | the image imagePullSecrets of plugins                      | `[]`                                         |
 
 ### clusterDefaultPool parameters

diff --git a/charts/spiderpool/crds/spiderpool.spidernet.io_spidermultusconfigs.yaml b/charts/spiderpool/crds/spiderpool.spidernet.io_spidermultusconfigs.yaml
@@ -43,6 +43,8 @@ spec:
                 - ipvlan
                 - sriov
                 - ovs
+                - ib-sriov
+                - ipoib
                 - custom
                 type: string
               coordinator:
@@ -96,6 +98,59 @@ spec:
               enableCoordinator:
                 default: true
                 type: boolean
+              ibsriov:
+                properties:
+                  ibKubernetesEnabled:
+                    default: false
+                    type: boolean
+                  ippools:
+                    description: SpiderpoolPools could specify the IPAM spiderpool
+                      CNI configuration default IPv4&IPv6 pools.
+                    properties:
+                      ipv4:
+                        items:
+                          type: string
+                        type: array
+                      ipv6:
+                        items:
+                          type: string
+                        type: array
+                    type: object
+                  linkState:
+                    default: enable
+                    enum:
+                    - auto
+                    - enable
+                    - disable
+                    type: string
+                  pkey:
+                    type: string
+                  rdmaIsolation:
+                    default: true
+                    type: boolean
+                  resourceName:
+                    type: string
+                required:
+                - resourceName
+                type: object
+              ipoib:
+                properties:
+                  ippools:
+                    description: SpiderpoolPools could specify the IPAM spiderpool
+                      CNI configuration default IPv4&IPv6 pools.
+                    properties:
+                      ipv4:
+                        items:
+                          type: string
+                        type: array
+                      ipv6:
+                        items:
+                          type: string
+                        type: array
+                    type: object
+                  master:
+                    type: string
+                type: object
               ipvlan:
                 properties:
                   bond:

diff --git a/charts/spiderpool/templates/daemonset.yaml b/charts/spiderpool/templates/daemonset.yaml
@@ -80,7 +80,7 @@ spec:
                       - linux
       {{- end }}
       initContainers:
-        {{- if or .Values.plugins.installCNI .Values.plugins.installRdmaCNI .Values.plugins.installOvsCNI }}
+        {{- if or .Values.plugins.installCNI .Values.plugins.installRdmaCNI .Values.plugins.installOvsCNI .Values.plugins.installibSriovCNI .Values.plugins.installIpoibCNI }}
         - name: install-plugins
           image: {{ include "plugins.image" . | quote }}
           imagePullPolicy: {{ .Values.plugins.image.pullPolicy }}
@@ -91,6 +91,10 @@ spec:
             value: {{ .Values.plugins.installOvsCNI | quote }}
           - name: INSTALL_RDMA_PLUGIN
             value: {{ .Values.plugins.installRdmaCNI | quote }}
+          - name: INSTALL_IB_SRIOV_PLUGIN
+            value: {{ .Values.plugins.installibSriovCNI | quote }}
+          - name: INSTALL_IPOIB_PLUGIN
+            value: {{ .Values.plugins.installIpoibCNI | quote }}
           command:
             - "/bin/sh"
             - "entrypoint.sh"

diff --git a/charts/spiderpool/values.yaml b/charts/spiderpool/values.yaml
@@ -265,10 +265,16 @@ plugins:
   installCNI: false
 
   ## @param plugins.installRdmaCNI install rdma cni used to isolate rdma device for sriov cni
-  installRdmaCNI: false
+  installRdmaCNI: true
 
   ## @param plugins.installOvsCNI install ovs cni to each node
-  installOvsCNI: false
+  installOvsCNI: true
+
+  ## @param plugins.installibSriovCNI install ib-sriov cni to each node
+  installibSriovCNI: true
+
+  ## @param plugins.installIpoibCNI install ipoib cni to each node
+  installIpoibCNI: true
 
   image:
     ## @param plugins.image.registry the image registry of plugins
@@ -284,7 +290,7 @@ plugins:
     digest: ""
 
     ## @param plugins.image.tag the image tag of plugins
-    tag: v0.8.0
+    tag: v0.9.2
 
     ## @param plugins.image.imagePullSecrets the image imagePullSecrets of plugins
     imagePullSecrets: []

diff --git a/cmd/spiderpool-init/cmd/config.go b/cmd/spiderpool-init/cmd/config.go
@@ -5,20 +5,16 @@ package cmd
 
 import (
 	"fmt"
-	"os"
-	"strconv"
-	"strings"
-
-	"k8s.io/utils/pointer"
-
+	"github.com/containernetworking/cni/libcni"
 	coordinatorcmd "github.com/spidernet-io/spiderpool/cmd/coordinator/cmd"
 	"github.com/spidernet-io/spiderpool/pkg/constant"
 	spiderpoolip "github.com/spidernet-io/spiderpool/pkg/ip"
 	spiderpoolv2beta1 "github.com/spidernet-io/spiderpool/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1"
-
-	"github.com/containernetworking/cni/libcni"
-	"github.com/spidernet-io/spiderpool/pkg/multuscniconfig"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/pointer"
+	"os"
+	"strconv"
+	"strings"
 )
 
 const (
@@ -310,7 +306,7 @@ func parseCNIFromConfig(cniConfigPath string) (string, string, error) {
 	var cniName, cniType string
 	if cniConfigPath == "" {
 		logger.Sugar().Infof("No network found in %s, create default multuscniconfig", cniConfigPath)
-		return "default", multuscniconfig.CustomType, nil
+		return "default", constant.CustomCNI, nil
 	}
 
 	logger.Sugar().Infof("the first cni config file is %s in /etc/cni/net.d", cniConfigPath)
@@ -332,14 +328,18 @@ func parseCNIFromConfig(cniConfigPath string) (string, string, error) {
 	}
 
 	switch cniType {
-	case multuscniconfig.MacVlanType:
-		cniType = multuscniconfig.MacVlanType
-	case multuscniconfig.IpVlanType:
-		cniType = multuscniconfig.IpVlanType
-	case multuscniconfig.SriovType:
-		cniType = multuscniconfig.SriovType
+	case constant.MacvlanCNI:
+		cniType = constant.MacvlanCNI
+	case constant.IPVlanCNI:
+		cniType = constant.IPVlanCNI
+	case constant.SriovCNI:
+		cniType = constant.SriovCNI
+	case constant.IBSriovCNI:
+		cniType = constant.IBSriovCNI
+	case constant.IPoIBCNI:
+		cniType = constant.IPoIBCNI
 	default:
-		cniType = multuscniconfig.CustomType
+		cniType = constant.CustomCNI
 	}
 
 	return cniName, cniType, nil