Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

CDI: two sriovnodepolicy configs(8 vfs) but only four deviceNode #735

Closed
cyclinder opened this issue Jul 16, 2024 · 7 comments
Closed

CDI: two sriovnodepolicy configs(8 vfs) but only four deviceNode #735

cyclinder opened this issue Jul 16, 2024 · 7 comments

Comments

@cyclinder
Copy link
Contributor

I have two sriovnodepolicy configs, see below:

root@controller-node-1:/home/cyclinder/sriov# kubectl get sriovnetworknodepolicies.sriovnetwork.openshift.io -A -o wide
NAMESPACE     NAME      AGE
kube-system   policy1   43m
kube-system   policy2   7m44s

root@controller-node-1:/home/cyclinder/sriov# kubectl get sriovnetworknodestates.sriovnetwork.openshift.io -n kube-system -o yaml
apiVersion: v1
items:
- apiVersion: sriovnetwork.openshift.io/v1
  kind: SriovNetworkNodeState
  metadata:
    annotations:
      sriovnetwork.openshift.io/current-state: Idle
      sriovnetwork.openshift.io/desired-state: Idle
    creationTimestamp: "2024-07-16T03:50:05Z"
    generation: 5
    name: worker-node-1
    namespace: kube-system
    ownerReferences:
    - apiVersion: sriovnetwork.openshift.io/v1
      blockOwnerDeletion: true
      controller: true
      kind: SriovOperatorConfig
      name: default
      uid: 2427ae73-ef95-4f57-aa85-c681ff9a48bb
    resourceVersion: "40147316"
    uid: 67f59ed6-85a3-4913-a1bf-3697dd008310
  spec:
    interfaces:
    - name: enp4s0f0np0
      numVfs: 4
      pciAddress: "0000:04:00.0"
      vfGroups:
      - isRdma: true
        policyName: policy1
        resourceName: rdma_resource
        vfRange: 0-3
    - name: enp4s0f1np1
      numVfs: 4
      pciAddress: "0000:04:00.1"
      vfGroups:
      - isRdma: true
        policyName: policy2
        resourceName: rdma_resource1
        vfRange: 0-3
  status:
    interfaces:
    - Vfs:
      - deviceID: "1018"
        driver: mlx5_core
        mac: e6:bc:60:22:14:6c
        mtu: 1500
        name: enp4s0f0v0
        pciAddress: "0000:04:00.2"
        vendor: 15b3
        vfID: 0
      - deviceID: "1018"
        driver: mlx5_core
        mac: a2:d7:89:ad:5d:b7
        mtu: 1500
        name: enp4s0f0v1
        pciAddress: "0000:04:00.3"
        vendor: 15b3
        vfID: 1
      - deviceID: "1018"
        driver: mlx5_core
        mac: d2:0b:3f:c9:ab:a4
        mtu: 1500
        name: enp4s0f0v2
        pciAddress: "0000:04:00.4"
        vendor: 15b3
        vfID: 2
      - deviceID: "1018"
        driver: mlx5_core
        mac: 4e:37:ab:b2:68:d7
        mtu: 1500
        name: enp4s0f0v3
        pciAddress: "0000:04:00.5"
        vendor: 15b3
        vfID: 3
      deviceID: "1017"
      driver: mlx5_core
      eSwitchMode: legacy
      linkSpeed: 25000 Mb/s
      linkType: ETH
      mac: 04:3f:72:d0:d2:b2
      mtu: 1500
      name: enp4s0f0np0
      numVfs: 4
      pciAddress: "0000:04:00.0"
      totalvfs: 4
      vendor: 15b3
    - Vfs:
      - deviceID: "1018"
        driver: mlx5_core
        mac: 3e:3a:7f:af:11:99
        mtu: 1500
        name: enp4s0f1v0
        pciAddress: "0000:04:00.6"
        vendor: 15b3
        vfID: 0
      - deviceID: "1018"
        driver: mlx5_core
        mac: 6e:c1:0e:52:ea:d8
        mtu: 1500
        name: enp4s0f1v1
        pciAddress: "0000:04:00.7"
        vendor: 15b3
        vfID: 1
      - deviceID: "1018"
        driver: mlx5_core
        mac: 8e:c8:1d:fc:69:0d
        mtu: 1500
        name: enp4s0f1v2
        pciAddress: "0000:04:01.0"
        vendor: 15b3
        vfID: 2
      - deviceID: "1018"
        driver: mlx5_core
        mac: 52:4c:5c:b1:1d:44
        mtu: 1500
        name: enp4s0f1v3
        pciAddress: "0000:04:01.1"
        vendor: 15b3
        vfID: 3
      deviceID: "1017"
      driver: mlx5_core
      eSwitchMode: legacy
      linkSpeed: 10000 Mb/s
      linkType: ETH
      mac: 04:3f:72:d0:d2:b3
      mtu: 1500
      name: enp4s0f1np1
      numVfs: 4
      pciAddress: "0000:04:00.1"
      totalvfs: 4
      vendor: 15b3
    syncStatus: Succeeded
kind: List
metadata:
  resourceVersion: ""

root@worker-node-1:~# cat /var/run/cdi/sriov-dp-spidernet.io.yaml
cdiVersion: 0.5.0
containerEdits: {}
devices:
- containerEdits:
    deviceNodes:
    - hostPath: /dev/infiniband/issm6
      path: /dev/infiniband/issm6
      permissions: rw
    - hostPath: /dev/infiniband/umad6
      path: /dev/infiniband/umad6
      permissions: rw
    - hostPath: /dev/infiniband/uverbs6
      path: /dev/infiniband/uverbs6
      permissions: rw
    - hostPath: /dev/infiniband/rdma_cm
      path: /dev/infiniband/rdma_cm
      permissions: rw
  name: "0000:04:00.6"
- containerEdits:
    deviceNodes:
    - hostPath: /dev/infiniband/issm7
      path: /dev/infiniband/issm7
      permissions: rw
    - hostPath: /dev/infiniband/umad7
      path: /dev/infiniband/umad7
      permissions: rw
    - hostPath: /dev/infiniband/uverbs7
      path: /dev/infiniband/uverbs7
      permissions: rw
    - hostPath: /dev/infiniband/rdma_cm
      path: /dev/infiniband/rdma_cm
      permissions: rw
  name: "0000:04:00.7"
- containerEdits:
    deviceNodes:
    - hostPath: /dev/infiniband/issm8
      path: /dev/infiniband/issm8
      permissions: rw
    - hostPath: /dev/infiniband/umad8
      path: /dev/infiniband/umad8
      permissions: rw
    - hostPath: /dev/infiniband/uverbs8
      path: /dev/infiniband/uverbs8
      permissions: rw
    - hostPath: /dev/infiniband/rdma_cm
      path: /dev/infiniband/rdma_cm
      permissions: rw
  name: "0000:04:01.0"
- containerEdits:
    deviceNodes:
    - hostPath: /dev/infiniband/issm9
      path: /dev/infiniband/issm9
      permissions: rw
    - hostPath: /dev/infiniband/umad9
      path: /dev/infiniband/umad9
      permissions: rw
    - hostPath: /dev/infiniband/uverbs9
      path: /dev/infiniband/uverbs9
      permissions: rw
    - hostPath: /dev/infiniband/rdma_cm
      path: /dev/infiniband/rdma_cm
      permissions: rw
  name: "0000:04:01.1"
kind: spidernet.io/net-pci
@zeeke
Copy link
Member

zeeke commented Jul 16, 2024

hi @cyclinder , I'm not getting the problem here. Policies are applied to the nodestate

interfaces:
    - name: enp4s0f0np0
      numVfs: 4
      pciAddress: "0000:04:00.0"
      vfGroups:
      - isRdma: true
        policyName: policy1
        resourceName: rdma_resource
        vfRange: 0-3
    - name: enp4s0f1np1
      numVfs: 4
      pciAddress: "0000:04:00.1"
      vfGroups:
      - isRdma: true
        policyName: policy2
        resourceName: rdma_resource1
        vfRange: 0-3

@cyclinder
Copy link
Contributor Author

We have 8 vfs, so we should also have 8 deviceNodes in the CDI file, right?

@adrianchiris
Copy link
Collaborator

@cyclinder are there any other files in /var/run/cdi or under /etc/cdi ?

also can you provide the following?

  • sriov-network-device-plugin config map generated by the operator ?
  • sriov-network-device-plugin daemonset yaml ?
  • sriovoperatorconfig CR ?

@cyclinder
Copy link
Contributor Author

are there any other files in /var/run/cdi or under /etc/cdi ?

No, only sriov-dp-spidernet.io.yaml file under the /var/run/cdi.

root@controller-node-1:/home/cyclinder/sriov# kubectl get configmaps -n kube-system device-plugin-config -o json | jq '.data'
{
  "worker-node-1": "{\"resourceList\":[{\"resourceName\":\"rdma_resource\",\"selectors\":{\"pfNames\":[\"enp4s0f0np0\"],\"IsRdma\":true,\"NeedVhostNet\":false},\"SelectorObj\":null},{\"resourceName\":\"rdma_resource1\",\"selectors\":{\"pfNames\":[\"enp4s0f1np1\"],\"IsRdma\":true,\"NeedVhostNet\":false},\"SelectorObj\":null}]}"
}

root@controller-node-1:/home/cyclinder/sriov# kubectl get ds -n kube-system sriov-device-plugin -o yaml
apiVersion: apps/v1
kind: DaemonSet
metadata:
  annotations:
    deprecated.daemonset.template.generation: "3"
    kubernetes.io/description: |
      This daemon set launches the SR-IOV network device plugin on each node.
    release.openshift.io/version: ""
  creationTimestamp: "2024-07-16T05:49:50Z"
  generation: 3
  name: sriov-device-plugin
  namespace: kube-system
  ownerReferences:
  - apiVersion: sriovnetwork.openshift.io/v1
    blockOwnerDeletion: true
    controller: true
    kind: SriovOperatorConfig
    name: default
    uid: 2427ae73-ef95-4f57-aa85-c681ff9a48bb
  resourceVersion: "40147369"
  uid: b6c01d09-98ed-4c9e-87bf-2d9f4cfa2096
spec:
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app: sriov-device-plugin
  template:
    metadata:
      creationTimestamp: null
      labels:
        app: sriov-device-plugin
        component: network
        openshift.io/component: network
        type: infra
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: node-role.kubernetes.io/worker
                operator: In
                values:
                - ""
            - matchExpressions:
              - key: node-role.kubernetes.io/worker
                operator: In
                values:
                - ""
      containers:
      - args:
        - --log-level=10
        - --resource-prefix=spidernet.io
        - --config-file=/etc/pcidp/$(NODE_NAME)
        - --use-cdi
        env:
        - name: NODE_NAME
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: spec.nodeName
        image: ghcr.m.daocloud.io/k8snetworkplumbingwg/sriov-network-device-plugin
        imagePullPolicy: Always
        name: sriov-device-plugin
        resources:
          requests:
            cpu: 10m
            memory: 50Mi
        securityContext:
          privileged: true
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
        volumeMounts:
        - mountPath: /var/lib/kubelet/device-plugins
          name: devicesock
        - mountPath: /var/lib/kubelet/plugins_registry
          name: plugins-registry
        - mountPath: /etc/pcidp/
          name: config-volume
          readOnly: true
        - mountPath: /var/run/k8s.cni.cncf.io/devinfo/dp
          name: device-info
        - mountPath: /var/run/cdi
          name: dynamic-cdi
      dnsPolicy: ClusterFirst
      hostNetwork: true
      nodeSelector:
        kubernetes.io/os: linux
        node-role.kubernetes.io/worker: ""
      priorityClassName: system-node-critical
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext: {}
      serviceAccount: sriov-device-plugin
      serviceAccountName: sriov-device-plugin
      terminationGracePeriodSeconds: 30
      tolerations:
      - operator: Exists
      volumes:
      - hostPath:
          path: /var/lib/kubelet/device-plugins
          type: ""
        name: devicesock
      - hostPath:
          path: /var/lib/kubelet/plugins_registry
          type: ""
        name: plugins-registry
      - configMap:
          defaultMode: 420
          name: device-plugin-config
        name: config-volume
      - hostPath:
          path: /var/run/k8s.cni.cncf.io/devinfo/dp
          type: DirectoryOrCreate
        name: device-info
      - hostPath:
          path: /var/run/cdi
          type: DirectoryOrCreate
        name: dynamic-cdi
  updateStrategy:
    rollingUpdate:
      maxSurge: 0
      maxUnavailable: 33%
    type: RollingUpdate

root@controller-node-1:/home/cyclinder/sriov# kubectl get sriovoperatorconfigs.sriovnetwork.openshift.io -n kube-system default -o yaml
apiVersion: sriovnetwork.openshift.io/v1
kind: SriovOperatorConfig
metadata:
  annotations:
    meta.helm.sh/release-name: sriov-operator
    meta.helm.sh/release-namespace: kube-system
  creationTimestamp: "2024-07-16T03:27:39Z"
  generation: 3
  labels:
    app.kubernetes.io/managed-by: Helm
  name: default
  namespace: kube-system
  resourceVersion: "40134294"
  uid: 2427ae73-ef95-4f57-aa85-c681ff9a48bb
spec:
  configurationMode: daemon
  disableDrain: false
  enableInjector: false
  enableOperatorWebhook: false
  logLevel: 2
  useCDI: true

@adrianchiris
Copy link
Collaborator

adrianchiris commented Jul 16, 2024 •
edited

a couple more questions @cyclinder :

if you restart device plugin will CDI file contain all devices ?
if you put both PFs under the same resource ? (i.e same policy) will you then have all 7 VFs ?

im thinking sriov-device-plugin doesnt handle merging cdi spec of multiple resources well.

if thats the case can you create an issue in sriov-network-device-plugin ?

@cyclinder
Copy link
Contributor Author

thanks @adrianchiris!

if you restart device plugin will CDI file contain all devices ?

Still no

if you put both PFs under the same resource ? (i.e same policy) will you then have all 7 VFs ?

Yes.

if thats the case can you create an issue in sriov-network-device-plugin ?

Sure, I would create an issue for this in sriov-network-device-plugin.

@cyclinder cyclinder mentioned this issue Jul 19, 2024
Open
@adrianchiris
Copy link
Collaborator

Great, thx @cyclinder closing this one

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees
No one assigned
Labels
None yet
Projects
None yet
Milestone
No milestone
Development

No branches or pull requests
3 participants
@zeeke @adrianchiris @cyclinder