From 700ca027a18fd7d4531f91ddf9d1823cce60dfbe Mon Sep 17 00:00:00 2001 From: Yury Kulazhenkov Date: Fri, 5 Apr 2024 16:09:35 +0300 Subject: [PATCH 1/2] Report VF representor name in the status field Add support for reporting VF representor name when the NIC is in switchev mode Signed-off-by: Yury Kulazhenkov --- go.mod | 3 +- go.sum | 6 +- pkg/helper/mock/mock_helper.go | 15 - .../lib/sriovnet/mock/mock_sriovnet.go | 49 ++ pkg/host/internal/lib/sriovnet/sriovnet.go | 22 + pkg/host/internal/sriov/sriov.go | 37 +- pkg/host/internal/sriov/sriov_test.go | 6 +- pkg/host/manager.go | 4 +- pkg/host/mock/mock_host.go | 15 - pkg/host/types/interfaces.go | 3 - .../sriovnet/.golangci.yml | 116 ++++ .../k8snetworkplumbingwg/sriovnet/LICENSE | 201 +++++++ .../k8snetworkplumbingwg/sriovnet/Makefile | 63 +++ .../k8snetworkplumbingwg/sriovnet/README.md | 60 +++ .../sriovnet/file_access.go | 139 +++++ .../sriovnet/mofed_ib_helper.go | 57 ++ .../pkg/utils/filesystem/defaultfs.go | 132 +++++ .../sriovnet/pkg/utils/filesystem/fakefs.go | 151 ++++++ .../pkg/utils/filesystem/filesystem.go | 41 ++ .../pkg/utils/netlinkops/netlinkops.go | 114 ++++ .../k8snetworkplumbingwg/sriovnet/sriovnet.go | 488 +++++++++++++++++ .../sriovnet/sriovnet_aux.go | 111 ++++ .../sriovnet/sriovnet_helper.go | 130 +++++ .../sriovnet/sriovnet_switchdev.go | 499 ++++++++++++++++++ .../k8snetworkplumbingwg/sriovnet/utils.go | 45 ++ vendor/github.com/spf13/afero/afero.go | 2 +- vendor/github.com/spf13/afero/basepath.go | 1 - .../github.com/spf13/afero/copyOnWriteFs.go | 9 +- vendor/github.com/spf13/afero/ioutil.go | 11 +- vendor/github.com/spf13/afero/mem/file.go | 7 +- vendor/github.com/spf13/afero/memmap.go | 18 +- vendor/github.com/spf13/afero/regexpfs.go | 1 - vendor/github.com/spf13/afero/symlink.go | 6 +- vendor/github.com/spf13/afero/unionFile.go | 5 +- vendor/github.com/spf13/afero/util.go | 7 +- vendor/modules.txt | 7 +- 36 files changed, 2504 insertions(+), 77 deletions(-) create mode 100644 pkg/host/internal/lib/sriovnet/mock/mock_sriovnet.go create mode 100644 pkg/host/internal/lib/sriovnet/sriovnet.go create mode 100644 vendor/github.com/k8snetworkplumbingwg/sriovnet/.golangci.yml create mode 100644 vendor/github.com/k8snetworkplumbingwg/sriovnet/LICENSE create mode 100644 vendor/github.com/k8snetworkplumbingwg/sriovnet/Makefile create mode 100644 vendor/github.com/k8snetworkplumbingwg/sriovnet/README.md create mode 100644 vendor/github.com/k8snetworkplumbingwg/sriovnet/file_access.go create mode 100644 vendor/github.com/k8snetworkplumbingwg/sriovnet/mofed_ib_helper.go create mode 100644 vendor/github.com/k8snetworkplumbingwg/sriovnet/pkg/utils/filesystem/defaultfs.go create mode 100644 vendor/github.com/k8snetworkplumbingwg/sriovnet/pkg/utils/filesystem/fakefs.go create mode 100644 vendor/github.com/k8snetworkplumbingwg/sriovnet/pkg/utils/filesystem/filesystem.go create mode 100644 vendor/github.com/k8snetworkplumbingwg/sriovnet/pkg/utils/netlinkops/netlinkops.go create mode 100644 vendor/github.com/k8snetworkplumbingwg/sriovnet/sriovnet.go create mode 100644 vendor/github.com/k8snetworkplumbingwg/sriovnet/sriovnet_aux.go create mode 100644 vendor/github.com/k8snetworkplumbingwg/sriovnet/sriovnet_helper.go create mode 100644 vendor/github.com/k8snetworkplumbingwg/sriovnet/sriovnet_switchdev.go create mode 100644 vendor/github.com/k8snetworkplumbingwg/sriovnet/utils.go diff --git a/go.mod b/go.mod index fca66390c..c875f75cb 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/jaypipes/ghw v0.9.0 github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0 github.com/k8snetworkplumbingwg/sriov-network-device-plugin v0.0.0-20221127172732-a5a7395122e3 + github.com/k8snetworkplumbingwg/sriovnet v1.2.0 github.com/onsi/ginkgo/v2 v2.11.0 github.com/onsi/gomega v1.27.10 github.com/openshift-kni/k8sreporter v1.0.4 @@ -115,7 +116,7 @@ require ( github.com/prometheus/procfs v0.12.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/shopspring/decimal v1.2.0 // indirect - github.com/spf13/afero v1.9.3 // indirect + github.com/spf13/afero v1.9.4 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace // indirect github.com/stretchr/objx v0.5.0 // indirect diff --git a/go.sum b/go.sum index 30f10a74f..85359272d 100644 --- a/go.sum +++ b/go.sum @@ -300,6 +300,8 @@ github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0 h1:V github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0/go.mod h1:nqCI7aelBJU61wiBeeZWJ6oi4bJy5nrjkM6lWIMA4j0= github.com/k8snetworkplumbingwg/sriov-network-device-plugin v0.0.0-20221127172732-a5a7395122e3 h1:hIHzF4vNTCFb9UMngrcJAMvdNslCekaSvNbfQt21CUw= github.com/k8snetworkplumbingwg/sriov-network-device-plugin v0.0.0-20221127172732-a5a7395122e3/go.mod h1:b0YSmUuNOy6CkEmV27XfmZ3a7njs2pjxHoFAvfLbUII= +github.com/k8snetworkplumbingwg/sriovnet v1.2.0 h1:6ELfAxCB1dvosGUy3DVRmfH+HWTzmPD3W67HKQvMR1M= +github.com/k8snetworkplumbingwg/sriovnet v1.2.0/go.mod h1:jyWzGe6ZtYiPq6ih6aXCOy6mZ49Y9mNyBOLBBXnli+k= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= @@ -398,8 +400,8 @@ github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFR github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU= github.com/spf13/afero v1.4.1/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= -github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= +github.com/spf13/afero v1.9.4 h1:Sd43wM1IWz/s1aVXdOBkjJvuP8UdyqioeE4AmM0QsBs= +github.com/spf13/afero v1.9.4/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= diff --git a/pkg/helper/mock/mock_helper.go b/pkg/helper/mock/mock_helper.go index 4d8db7bf6..797143096 100644 --- a/pkg/helper/mock/mock_helper.go +++ b/pkg/helper/mock/mock_helper.go @@ -8,7 +8,6 @@ import ( reflect "reflect" gomock "github.com/golang/mock/gomock" - ghw "github.com/jaypipes/ghw" v1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" store "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/store" types "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types" @@ -516,20 +515,6 @@ func (mr *MockHostHelpersInterfaceMockRecorder) GetPhysSwitchID(name interface{} return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPhysSwitchID", reflect.TypeOf((*MockHostHelpersInterface)(nil).GetPhysSwitchID), name) } -// GetVfInfo mocks base method. -func (m *MockHostHelpersInterface) GetVfInfo(pciAddr string, devices []*ghw.PCIDevice) v1.VirtualFunction { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetVfInfo", pciAddr, devices) - ret0, _ := ret[0].(v1.VirtualFunction) - return ret0 -} - -// GetVfInfo indicates an expected call of GetVfInfo. -func (mr *MockHostHelpersInterfaceMockRecorder) GetVfInfo(pciAddr, devices interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVfInfo", reflect.TypeOf((*MockHostHelpersInterface)(nil).GetVfInfo), pciAddr, devices) -} - // HasDriver mocks base method. func (m *MockHostHelpersInterface) HasDriver(pciAddr string) (bool, string) { m.ctrl.T.Helper() diff --git a/pkg/host/internal/lib/sriovnet/mock/mock_sriovnet.go b/pkg/host/internal/lib/sriovnet/mock/mock_sriovnet.go new file mode 100644 index 000000000..937ce7914 --- /dev/null +++ b/pkg/host/internal/lib/sriovnet/mock/mock_sriovnet.go @@ -0,0 +1,49 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: sriovnet.go + +// Package mock_sriovnet is a generated GoMock package. +package mock_sriovnet + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" +) + +// MockSriovnetLib is a mock of SriovnetLib interface. +type MockSriovnetLib struct { + ctrl *gomock.Controller + recorder *MockSriovnetLibMockRecorder +} + +// MockSriovnetLibMockRecorder is the mock recorder for MockSriovnetLib. +type MockSriovnetLibMockRecorder struct { + mock *MockSriovnetLib +} + +// NewMockSriovnetLib creates a new mock instance. +func NewMockSriovnetLib(ctrl *gomock.Controller) *MockSriovnetLib { + mock := &MockSriovnetLib{ctrl: ctrl} + mock.recorder = &MockSriovnetLibMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockSriovnetLib) EXPECT() *MockSriovnetLibMockRecorder { + return m.recorder +} + +// GetVfRepresentor mocks base method. +func (m *MockSriovnetLib) GetVfRepresentor(uplink string, vfIndex int) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetVfRepresentor", uplink, vfIndex) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetVfRepresentor indicates an expected call of GetVfRepresentor. +func (mr *MockSriovnetLibMockRecorder) GetVfRepresentor(uplink, vfIndex interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVfRepresentor", reflect.TypeOf((*MockSriovnetLib)(nil).GetVfRepresentor), uplink, vfIndex) +} diff --git a/pkg/host/internal/lib/sriovnet/sriovnet.go b/pkg/host/internal/lib/sriovnet/sriovnet.go new file mode 100644 index 000000000..7318e77ba --- /dev/null +++ b/pkg/host/internal/lib/sriovnet/sriovnet.go @@ -0,0 +1,22 @@ +package sriovnet + +import ( + "github.com/k8snetworkplumbingwg/sriovnet" +) + +func New() SriovnetLib { + return &libWrapper{} +} + +//go:generate ../../../../../bin/mockgen -destination mock/mock_sriovnet.go -source sriovnet.go +type SriovnetLib interface { + // GetVfRepresentor returns representor name for VF device + GetVfRepresentor(uplink string, vfIndex int) (string, error) +} + +type libWrapper struct{} + +// GetVfRepresentor returns representor name for VF device +func (w *libWrapper) GetVfRepresentor(pfName string, vfIndex int) (string, error) { + return sriovnet.GetVfRepresentor(pfName, vfIndex) +} diff --git a/pkg/host/internal/sriov/sriov.go b/pkg/host/internal/sriov/sriov.go index 0e2bfa531..fef9f3a69 100644 --- a/pkg/host/internal/sriov/sriov.go +++ b/pkg/host/internal/sriov/sriov.go @@ -19,6 +19,7 @@ import ( "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" dputilsPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/dputils" netlinkPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink" + sriovnetPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/sriovnet" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/store" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" @@ -38,6 +39,7 @@ type sriov struct { vdpaHelper types.VdpaInterface netlinkLib netlinkPkg.NetlinkLib dputilsLib dputilsPkg.DPUtilsLib + sriovnetLib sriovnetPkg.SriovnetLib } func New(utilsHelper utils.CmdInterface, @@ -46,7 +48,8 @@ func New(utilsHelper utils.CmdInterface, udevHelper types.UdevInterface, vdpaHelper types.VdpaInterface, netlinkLib netlinkPkg.NetlinkLib, - dputilsLib dputilsPkg.DPUtilsLib) types.SriovInterface { + dputilsLib dputilsPkg.DPUtilsLib, + sriovnetLib sriovnetPkg.SriovnetLib) types.SriovInterface { return &sriov{utilsHelper: utilsHelper, kernelHelper: kernelHelper, networkHelper: networkHelper, @@ -54,6 +57,7 @@ func New(utilsHelper utils.CmdInterface, vdpaHelper: vdpaHelper, netlinkLib: netlinkLib, dputilsLib: dputilsLib, + sriovnetLib: sriovnetLib, } } @@ -108,26 +112,35 @@ func (s *sriov) ResetSriovDevice(ifaceStatus sriovnetworkv1.InterfaceExt) error return nil } -func (s *sriov) GetVfInfo(pciAddr string, devices []*ghw.PCIDevice) sriovnetworkv1.VirtualFunction { - driver, err := s.dputilsLib.GetDriverName(pciAddr) +func (s *sriov) getVfInfo(vfAddr string, pfName string, eswitchMode string, devices []*ghw.PCIDevice) sriovnetworkv1.VirtualFunction { + driver, err := s.dputilsLib.GetDriverName(vfAddr) if err != nil { - log.Log.Error(err, "getVfInfo(): unable to parse device driver", "device", pciAddr) + log.Log.Error(err, "getVfInfo(): unable to parse device driver", "device", vfAddr) } - id, err := s.dputilsLib.GetVFID(pciAddr) + id, err := s.dputilsLib.GetVFID(vfAddr) if err != nil { - log.Log.Error(err, "getVfInfo(): unable to get VF index", "device", pciAddr) + log.Log.Error(err, "getVfInfo(): unable to get VF index", "device", vfAddr) } vf := sriovnetworkv1.VirtualFunction{ - PciAddress: pciAddr, + PciAddress: vfAddr, Driver: driver, VfID: id, - VdpaType: s.vdpaHelper.DiscoverVDPAType(pciAddr), + VdpaType: s.vdpaHelper.DiscoverVDPAType(vfAddr), } - if name := s.networkHelper.TryGetInterfaceName(pciAddr); name != "" { + if eswitchMode == sriovnetworkv1.ESwithModeSwitchDev { + repName, err := s.sriovnetLib.GetVfRepresentor(pfName, id) + if err != nil { + log.Log.Error(err, "getVfInfo(): failed to get VF representor name", "device", vfAddr) + } else { + vf.RepresentorName = repName + } + } + + if name := s.networkHelper.TryGetInterfaceName(vfAddr); name != "" { link, err := s.netlinkLib.LinkByName(name) if err != nil { - log.Log.Error(err, "getVfInfo(): unable to get VF Link Object", "name", name, "device", pciAddr) + log.Log.Error(err, "getVfInfo(): unable to get VF Link Object", "name", name, "device", vfAddr) } else { vf.Name = name vf.Mtu = link.Attrs().MTU @@ -136,7 +149,7 @@ func (s *sriov) GetVfInfo(pciAddr string, devices []*ghw.PCIDevice) sriovnetwork } for _, device := range devices { - if pciAddr == device.Address { + if vfAddr == device.Address { vf.Vendor = device.Vendor.ID vf.DeviceID = device.Product.ID break @@ -291,7 +304,7 @@ func (s *sriov) DiscoverSriovDevices(storeManager store.ManagerInterface) ([]sri continue } for _, vf := range vfs { - instance := s.GetVfInfo(vf, devices) + instance := s.getVfInfo(vf, pfNetName, iface.EswitchMode, devices) iface.VFs = append(iface.VFs, instance) } } diff --git a/pkg/host/internal/sriov/sriov_test.go b/pkg/host/internal/sriov/sriov_test.go index 15faebe9c..563acb1fa 100644 --- a/pkg/host/internal/sriov/sriov_test.go +++ b/pkg/host/internal/sriov/sriov_test.go @@ -15,6 +15,7 @@ import ( sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" dputilsMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/dputils/mock" netlinkMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink/mock" + sriovnetMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/sriovnet/mock" hostMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/mock" hostStoreMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/store/mock" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types" @@ -27,6 +28,7 @@ var _ = Describe("SRIOV", func() { s types.SriovInterface netlinkLibMock *netlinkMockPkg.MockNetlinkLib dputilsLibMock *dputilsMockPkg.MockDPUtilsLib + sriovnetLibMock *sriovnetMockPkg.MockSriovnetLib hostMock *hostMockPkg.MockHostManagerInterface storeManagerMode *hostStoreMockPkg.MockManagerInterface @@ -38,10 +40,12 @@ var _ = Describe("SRIOV", func() { testCtrl = gomock.NewController(GinkgoT()) netlinkLibMock = netlinkMockPkg.NewMockNetlinkLib(testCtrl) dputilsLibMock = dputilsMockPkg.NewMockDPUtilsLib(testCtrl) + sriovnetLibMock = sriovnetMockPkg.NewMockSriovnetLib(testCtrl) + hostMock = hostMockPkg.NewMockHostManagerInterface(testCtrl) storeManagerMode = hostStoreMockPkg.NewMockManagerInterface(testCtrl) - s = New(nil, hostMock, hostMock, hostMock, hostMock, netlinkLibMock, dputilsLibMock) + s = New(nil, hostMock, hostMock, hostMock, hostMock, netlinkLibMock, dputilsLibMock, sriovnetLibMock) }) AfterEach(func() { diff --git a/pkg/host/manager.go b/pkg/host/manager.go index a79331b53..e640f367e 100644 --- a/pkg/host/manager.go +++ b/pkg/host/manager.go @@ -5,6 +5,7 @@ import ( "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/dputils" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/ethtool" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/sriovnet" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/network" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/service" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/sriov" @@ -40,12 +41,13 @@ func NewHostManager(utilsInterface utils.CmdInterface) HostManagerInterface { dpUtils := dputils.New() netlinkLib := netlink.New() ethtoolLib := ethtool.New() + sriovnetLib := sriovnet.New() k := kernel.New(utilsInterface) n := network.New(utilsInterface, dpUtils, netlinkLib, ethtoolLib) sv := service.New(utilsInterface) u := udev.New(utilsInterface) v := vdpa.New(k, netlinkLib) - sr := sriov.New(utilsInterface, k, n, u, v, netlinkLib, dpUtils) + sr := sriov.New(utilsInterface, k, n, u, v, netlinkLib, dpUtils, sriovnetLib) return &hostManager{ utilsInterface, diff --git a/pkg/host/mock/mock_host.go b/pkg/host/mock/mock_host.go index 4ea2f0e8b..77245a332 100644 --- a/pkg/host/mock/mock_host.go +++ b/pkg/host/mock/mock_host.go @@ -8,7 +8,6 @@ import ( reflect "reflect" gomock "github.com/golang/mock/gomock" - ghw "github.com/jaypipes/ghw" v1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" store "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/store" types "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types" @@ -440,20 +439,6 @@ func (mr *MockHostManagerInterfaceMockRecorder) GetPhysSwitchID(name interface{} return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPhysSwitchID", reflect.TypeOf((*MockHostManagerInterface)(nil).GetPhysSwitchID), name) } -// GetVfInfo mocks base method. -func (m *MockHostManagerInterface) GetVfInfo(pciAddr string, devices []*ghw.PCIDevice) v1.VirtualFunction { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetVfInfo", pciAddr, devices) - ret0, _ := ret[0].(v1.VirtualFunction) - return ret0 -} - -// GetVfInfo indicates an expected call of GetVfInfo. -func (mr *MockHostManagerInterfaceMockRecorder) GetVfInfo(pciAddr, devices interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVfInfo", reflect.TypeOf((*MockHostManagerInterface)(nil).GetVfInfo), pciAddr, devices) -} - // HasDriver mocks base method. func (m *MockHostManagerInterface) HasDriver(pciAddr string) (bool, string) { m.ctrl.T.Helper() diff --git a/pkg/host/types/interfaces.go b/pkg/host/types/interfaces.go index 48d47b424..d47874792 100644 --- a/pkg/host/types/interfaces.go +++ b/pkg/host/types/interfaces.go @@ -1,7 +1,6 @@ package types import ( - "github.com/jaypipes/ghw" "github.com/vishvananda/netlink" sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" @@ -130,8 +129,6 @@ type SriovInterface interface { // SetSriovNumVfs changes the number of virtual functions allocated for a specific // physical function base on pci address SetSriovNumVfs(pciAddr string, numVfs int) error - // GetVfInfo returns the virtual function information is the operator struct from the host information - GetVfInfo(pciAddr string, devices []*ghw.PCIDevice) sriovnetworkv1.VirtualFunction // SetVfGUID sets the GUID for a virtual function SetVfGUID(vfAddr string, pfLink netlink.Link) error // VFIsReady returns the interface virtual function if the device is ready diff --git a/vendor/github.com/k8snetworkplumbingwg/sriovnet/.golangci.yml b/vendor/github.com/k8snetworkplumbingwg/sriovnet/.golangci.yml new file mode 100644 index 000000000..64dbb3614 --- /dev/null +++ b/vendor/github.com/k8snetworkplumbingwg/sriovnet/.golangci.yml @@ -0,0 +1,116 @@ +run: + timeout: 10m + + # If set we pass it to "go list -mod={option}". From "go help modules": + # If invoked with -mod=readonly, the go command is disallowed from the implicit + # automatic updating of go.mod described above. Instead, it fails when any changes + # to go.mod are needed. This setting is most useful to check that go.mod does + # not need updates, such as in a continuous integration and testing system. + # If invoked with -mod=vendor, the go command assumes that the vendor + # directory holds the correct copies of dependencies and ignores + # the dependency descriptions in go.mod. + # + # Allowed values: readonly|vendor|mod + # By default, it isn't set. + modules-download-mode: readonly + tests: false + +linters-settings: + dupl: + threshold: 150 + funlen: + lines: 100 + statements: 50 + goconst: + min-len: 2 + min-occurrences: 2 + gocritic: + enabled-tags: + - diagnostic + - experimental + - opinionated + - performance + - style + disabled-checks: + - dupImport # https://github.com/go-critic/go-critic/issues/845 + - ifElseChain + - octalLiteral + - whyNoLint + - wrapperFunc + - unnamedResult + gocognit: + min-complexity: 30 + goimports: + local-prefixes: github.com/k8snetworkplumbingwg/sriovnet + golint: + min-confidence: 0 + gomnd: + settings: + mnd: + # don't include the "operation" and "assign" + checks: argument,case,condition,return + ignored-numbers: "1,2,10,32" + govet: + check-shadowing: true + settings: + printf: + funcs: + - (github.com/rs/zerolog/zerolog.Event).Msgf + lll: + line-length: 120 + misspell: + locale: US + ignore-words: + - flavour + - flavours + prealloc: + # Report preallocation suggestions only on simple loops that have no returns/breaks/continues/gotos in them. + # True by default. + simple: true + range-loops: true # Report preallocation suggestions on range loops, true by default + for-loops: false # Report preallocation suggestions on for loops, false by default + +linters: + # please, do not use `enable-all`: it's deprecated and will be removed soon. + # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint + disable-all: true + enable: + - bodyclose + - depguard + - dogsled + - dupl + - errcheck + - funlen + - gochecknoinits + - goconst + - gocritic + - gocognit + - gofmt + - goimports + - gomnd + - goprintffuncname + - gosec + - gosimple + - govet + - ineffassign + - lll + - misspell + - nakedret + - prealloc + - revive + - rowserrcheck + - exportloopref + - staticcheck + - stylecheck + - typecheck + - unconvert + - unparam + - unused + - whitespace + +issues: + # Excluding configuration per-path, per-linter, per-text and per-source + exclude-rules: + - text: "Magic number: 1" + linters: + - gomnd diff --git a/vendor/github.com/k8snetworkplumbingwg/sriovnet/LICENSE b/vendor/github.com/k8snetworkplumbingwg/sriovnet/LICENSE new file mode 100644 index 000000000..261eeb9e9 --- /dev/null +++ b/vendor/github.com/k8snetworkplumbingwg/sriovnet/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/k8snetworkplumbingwg/sriovnet/Makefile b/vendor/github.com/k8snetworkplumbingwg/sriovnet/Makefile new file mode 100644 index 000000000..180a8a809 --- /dev/null +++ b/vendor/github.com/k8snetworkplumbingwg/sriovnet/Makefile @@ -0,0 +1,63 @@ +# Package related +PACKAGE := sriovnet +BIN_DIR := $(CURDIR)/bin +GOFILES := $(shell find . -name "*.go" | grep -vE "(\/vendor\/)|(_test.go)") +PKGS := $(or $(PKG),$(shell go list ./... | grep -v "^$(PACKAGE)/vendor/")) +TESTPKGS := $(shell go list -f '{{ if or .TestGoFiles .XTestGoFiles }}{{ .ImportPath }}{{ end }}' $(PKGS)) + +# Go tools +GOLANGCI_LINT := $(BIN_DIR)/golangci-lint +GCOV2LCOV := $(BIN_DIR)/gcov2lcov +# golangci-lint version should be updated periodically +# we keep it fixed to avoid it from unexpectedly failing on the project +# in case of a version bump +GOLANGCI_LINT_VER := v1.49.0 + +Q = $(if $(filter 1,$V),,@) + +.PHONY: all +all: lint test build + +$(BIN_DIR): + @mkdir -p $@ + +build: $(GOFILES) ;@ ## build sriovnet + @CGO_ENABLED=0 go build -v + +# Tests + +.PHONY: lint +lint: | $(GOLANGCI_LINT) ; $(info running golangci-lint...) @ ## Run lint tests + $Q $(GOLANGCI_LINT) run + +.PHONY: test tests +test: ; $(info running unit tests...) ## Run unit tests + $Q go test ./... + +tests: test lint ; ## Run all tests + +COVERAGE_MODE = count +.PHONY: test-coverage test-coverage-tools +test-coverage-tools: $(GCOV2LCOV) +test-coverage: | test-coverage-tools; $(info running coverage tests...) @ ## Run coverage tests + $Q go test -covermode=$(COVERAGE_MODE) -coverprofile=sriovnet.cover ./... + $Q $(GCOV2LCOV) -infile sriovnet.cover -outfile sriovnet.info + +# Tools +$(GOLANGCI_LINT): | $(BIN_DIR) ; $(info building golangci-lint...) + $Q GOBIN=$(BIN_DIR) go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VER) + +$(GCOV2LCOV): | $(BIN_DIR) ; $(info building gocov2lcov...) + $Q GOBIN=$(BIN_DIR) go install github.com/jandelgado/gcov2lcov@v1.0.5 + +# Misc +.PHONY: clean +clean: ; $(info Cleaning...) @ ## Cleanup everything + @rm -rf $(BIN_DIR) + @rm sriovnet.cover + @rm sriovnet.info + +.PHONY: help +help: ; @ ## Show this message + @grep -E '^[ a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ + awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' diff --git a/vendor/github.com/k8snetworkplumbingwg/sriovnet/README.md b/vendor/github.com/k8snetworkplumbingwg/sriovnet/README.md new file mode 100644 index 000000000..2679318aa --- /dev/null +++ b/vendor/github.com/k8snetworkplumbingwg/sriovnet/README.md @@ -0,0 +1,60 @@ +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](http://www.apache.org/licenses/LICENSE-2.0) +[![Go Report Card](https://goreportcard.com/badge/github.com/k8snetworkplumbingwg/sriovnet)](https://goreportcard.com/report/github.com/k8snetworkplumbingwg/sriovnet) +[![Build](https://github.com/k8snetworkplumbingwg/sriovnet/actions/workflows/build.yaml/badge.svg)](https://github.com/k8snetworkplumbingwg/sriovnet/actions/workflows/build.yaml) +[![Test](https://github.com/k8snetworkplumbingwg/sriovnet/actions/workflows/test.yaml/badge.svg)](https://github.com/k8snetworkplumbingwg/sriovnet/actions/workflows/test.yaml) +[![Coverage Status](https://coveralls.io/repos/github/k8snetworkplumbingwg/sriovnet/badge.svg)](https://coveralls.io/k8snetworkplumbingwg/sriovnet) + +# sriovnet +Go library to configure SRIOV networking devices + +Local build and test + +You can use go get command: +``` +go get github.com/k8snetworkplumbingwg/sriovnet +``` + +Example: + +```go +package main + +import ( + "fmt" + + "github.com/k8snetworkplumbingwg/sriovnet" +) + +func main() { + var vfList[10] *sriovnet.VfObj + + err1 := sriovnet.EnableSriov("ib0") + if err1 != nil { + return + } + + handle, err2 := sriovnet.GetPfNetdevHandle("ib0") + if err2 != nil { + return + } + err3 := sriovnet.ConfigVfs(handle, false) + if err3 != nil { + return + } + for i := 0; i < 10; i++ { + vfList[i], _ = sriovnet.AllocateVf(handle) + } + for _, vf := range handle.List { + fmt.Printf("after allocation vf = %v\n", vf) + } + for i := 0; i < 10; i++ { + if vfList[i] == nil { + continue + } + sriovnet.FreeVf(handle, vfList[i]) + } + for _, vf := range handle.List { + fmt.Printf("after free vf = %v\n", vf) + } +} +``` diff --git a/vendor/github.com/k8snetworkplumbingwg/sriovnet/file_access.go b/vendor/github.com/k8snetworkplumbingwg/sriovnet/file_access.go new file mode 100644 index 000000000..b0fe653b3 --- /dev/null +++ b/vendor/github.com/k8snetworkplumbingwg/sriovnet/file_access.go @@ -0,0 +1,139 @@ +//nolint:gomnd +package sriovnet + +import ( + "io" + "os" + "strconv" + "strings" + "syscall" +) + +type fileObject struct { + Path string + File *os.File +} + +func (attrib *fileObject) Exists() bool { + return fileExists(attrib.Path) +} + +func (attrib *fileObject) Open() (err error) { + attrib.File, err = os.OpenFile(attrib.Path, os.O_RDWR|syscall.O_NONBLOCK, 0660) + return err +} + +func (attrib *fileObject) OpenRO() (err error) { + attrib.File, err = os.OpenFile(attrib.Path, os.O_RDONLY, 0444) + return err +} + +func (attrib *fileObject) OpenWO() (err error) { + attrib.File, err = os.OpenFile(attrib.Path, os.O_WRONLY, 0444) + return err +} + +func (attrib *fileObject) Close() (err error) { + err = attrib.File.Close() + attrib.File = nil + return err +} + +func (attrib *fileObject) Read() (str string, err error) { + if attrib.File == nil { + err = attrib.OpenRO() + if err != nil { + return + } + defer func() { + e := attrib.Close() + if err == nil { + err = e + } + }() + } + _, err = attrib.File.Seek(0, io.SeekStart) + if err != nil { + return "", err + } + data, err := io.ReadAll(attrib.File) + if err != nil { + return "", err + } + return string(data), nil +} + +func (attrib *fileObject) Write(value string) (err error) { + if attrib.File == nil { + err = attrib.OpenWO() + if err != nil { + return + } + defer func() { + e := attrib.Close() + if err == nil { + err = e + } + }() + } + _, err = attrib.File.Seek(0, io.SeekStart) + if err != nil { + return err + } + _, err = attrib.File.WriteString(value) + return err +} + +func (attrib *fileObject) ReadInt() (value int, err error) { + s, err := attrib.Read() + if err != nil { + return 0, err + } + s = strings.Trim(s, "\n") + value, err = strconv.Atoi(s) + if err != nil { + return 0, err + } + + return value, err +} + +func (attrib *fileObject) WriteInt(value int) (err error) { + return attrib.Write(strconv.Itoa(value)) +} + +func lsFilesWithPrefix(dir, filePrefix string, ignoreDir bool) ([]string, error) { + var desiredFiles []string + + f, err := os.Open(dir) + if err != nil { + return nil, err + } + defer f.Close() + fileInfos, err := f.Readdir(-1) + if err != nil { + return nil, err + } + + for i := range fileInfos { + if ignoreDir && fileInfos[i].IsDir() { + continue + } + + if filePrefix == "" || + strings.Contains(fileInfos[i].Name(), filePrefix) { + desiredFiles = append(desiredFiles, fileInfos[i].Name()) + } + } + return desiredFiles, nil +} + +func dirExists(dirname string) bool { + info, err := os.Stat(dirname) + return err == nil && info.IsDir() +} + +func fileExists(dirname string) bool { + info, err := os.Stat(dirname) + return err == nil && !info.IsDir() +} diff --git a/vendor/github.com/k8snetworkplumbingwg/sriovnet/mofed_ib_helper.go b/vendor/github.com/k8snetworkplumbingwg/sriovnet/mofed_ib_helper.go new file mode 100644 index 000000000..0e99e4191 --- /dev/null +++ b/vendor/github.com/k8snetworkplumbingwg/sriovnet/mofed_ib_helper.go @@ -0,0 +1,57 @@ +package sriovnet + +import ( + "net" + "path/filepath" + "strconv" +) + +const ( + ibSriovCfgDir = "sriov" + ibSriovNodeFile = "node" + ibSriovPortFile = "port" + ibSriovPortAdminFile = "policy" + ibSriovPortAdminStateFollow = "Follow" +) + +func ibGetPortAdminState(pfNetdevName string, vfIndex int) (string, error) { + path := filepath.Join( + NetSysDir, pfNetdevName, pcidevPrefix, ibSriovCfgDir, strconv.Itoa(vfIndex), ibSriovPortAdminFile) + adminStateFile := fileObject{ + Path: path, + } + + state, err := adminStateFile.Read() + if err != nil { + return "", err + } + return state, nil +} + +func ibSetPortAdminState(pfNetdevName string, vfIndex int, newState string) error { + path := filepath.Join( + NetSysDir, pfNetdevName, pcidevPrefix, ibSriovCfgDir, strconv.Itoa(vfIndex), ibSriovPortAdminFile) + adminStateFile := fileObject{ + Path: path, + } + + return adminStateFile.Write(newState) +} + +func ibSetNodeGUID(pfNetdevName string, vfIndex int, guid net.HardwareAddr) error { + path := filepath.Join(NetSysDir, pfNetdevName, pcidevPrefix, ibSriovCfgDir, strconv.Itoa(vfIndex), ibSriovNodeFile) + nodeGUIDFile := fileObject{ + Path: path, + } + kernelGUIDFormat := guid.String() + return nodeGUIDFile.Write(kernelGUIDFormat) +} + +func ibSetPortGUID(pfNetdevName string, vfIndex int, guid net.HardwareAddr) error { + path := filepath.Join(NetSysDir, pfNetdevName, pcidevPrefix, ibSriovCfgDir, strconv.Itoa(vfIndex), ibSriovPortFile) + portGUIDFile := fileObject{ + Path: path, + } + kernelGUIDFormat := guid.String() + return portGUIDFile.Write(kernelGUIDFormat) +} diff --git a/vendor/github.com/k8snetworkplumbingwg/sriovnet/pkg/utils/filesystem/defaultfs.go b/vendor/github.com/k8snetworkplumbingwg/sriovnet/pkg/utils/filesystem/defaultfs.go new file mode 100644 index 000000000..f092e86df --- /dev/null +++ b/vendor/github.com/k8snetworkplumbingwg/sriovnet/pkg/utils/filesystem/defaultfs.go @@ -0,0 +1,132 @@ +package filesystem + +import ( + "io/fs" + "os" + "path/filepath" + "time" +) + +// DefaultFs implements Filesystem using same-named functions from "os" and "io/ioutil" +type DefaultFs struct{} + +// Stat via os.Stat +func (DefaultFs) Stat(name string) (os.FileInfo, error) { + return os.Stat(name) +} + +// Create via os.Create +func (DefaultFs) Create(name string) (File, error) { + file, err := os.Create(name) + if err != nil { + return nil, err + } + return &defaultFile{file}, nil +} + +// Rename via os.Rename +func (DefaultFs) Rename(oldpath, newpath string) error { + return os.Rename(oldpath, newpath) +} + +// MkdirAll via os.MkdirAll +func (DefaultFs) MkdirAll(path string, perm os.FileMode) error { + return os.MkdirAll(path, perm) +} + +// Chtimes via os.Chtimes +func (DefaultFs) Chtimes(name string, atime, mtime time.Time) error { + return os.Chtimes(name, atime, mtime) +} + +// RemoveAll via os.RemoveAll +func (DefaultFs) RemoveAll(path string) error { + return os.RemoveAll(path) +} + +// Remove via os.RemoveAll +func (DefaultFs) Remove(name string) error { + return os.Remove(name) +} + +// Readlink via os.Readlink +func (DefaultFs) Readlink(name string) (string, error) { + return os.Readlink(name) +} + +// Symlink via os.Symlink +func (DefaultFs) Symlink(oldname, newname string) error { + return os.Symlink(oldname, newname) +} + +// ReadFile via ioutil.ReadFile +func (DefaultFs) ReadFile(filename string) ([]byte, error) { + return os.ReadFile(filename) +} + +// TempDir via ioutil.TempDir +func (DefaultFs) TempDir(dir, prefix string) (string, error) { + return os.MkdirTemp(dir, prefix) +} + +// TempFile via ioutil.TempFile +func (DefaultFs) TempFile(dir, prefix string) (File, error) { + file, err := os.CreateTemp(dir, prefix) + if err != nil { + return nil, err + } + return &defaultFile{file}, nil +} + +// ReadDir via os.ReadDir +func (DefaultFs) ReadDir(dirname string) ([]os.FileInfo, error) { + entries, err := os.ReadDir(dirname) + if err != nil { + return nil, err + } + + infos := make([]fs.FileInfo, 0, len(entries)) + for _, entry := range entries { + info, err := entry.Info() + if err != nil { + return nil, err + } + infos = append(infos, info) + } + return infos, nil +} + +// Walk via filepath.Walk +func (DefaultFs) Walk(root string, walkFn filepath.WalkFunc) error { + return filepath.Walk(root, walkFn) +} + +// WriteFile via ioutil.Writefile +func (DefaultFs) WriteFile(filename string, data []byte, perm os.FileMode) error { + return os.WriteFile(filename, data, perm) +} + +// defaultFile implements File using same-named functions from "os" +type defaultFile struct { + file *os.File +} + +// Name via os.File.Name +func (file *defaultFile) Name() string { + return file.file.Name() +} + +// Write via os.File.Write +func (file *defaultFile) Write(b []byte) (n int, err error) { + return file.file.Write(b) +} + +// Sync via os.File.Sync +func (file *defaultFile) Sync() error { + return file.file.Sync() +} + +// Close via os.File.Close +func (file *defaultFile) Close() error { + return file.file.Close() +} diff --git a/vendor/github.com/k8snetworkplumbingwg/sriovnet/pkg/utils/filesystem/fakefs.go b/vendor/github.com/k8snetworkplumbingwg/sriovnet/pkg/utils/filesystem/fakefs.go new file mode 100644 index 000000000..05e6a4ca9 --- /dev/null +++ b/vendor/github.com/k8snetworkplumbingwg/sriovnet/pkg/utils/filesystem/fakefs.go @@ -0,0 +1,151 @@ +//nolint:gomnd +package filesystem + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "github.com/spf13/afero" +) + +// FakeFs is implemented in terms of afero +type FakeFs struct { + a afero.Afero +} + +// NewFakeFs returns a fake Filesystem that exists at fakeFsRoot as its base path, useful for unit tests. +// Returns: Filesystem interface, teardown method (cleanup of provided root path) and error. +// teardown method should be called at the end of each test to ensure environment is left clean. +func NewFakeFs(fakeFsRoot string) (Filesystem, func(), error) { + _, err := os.Stat(fakeFsRoot) + // if fakeFsRoot dir exists remove it. + if err == nil { + err = os.RemoveAll(fakeFsRoot) + if err != nil { + return nil, nil, fmt.Errorf("failed to cleanup fake root dir %s. %s", fakeFsRoot, err) + } + } else if !os.IsNotExist(err) { + return nil, nil, fmt.Errorf("failed to lstat fake root dir %s. %s", fakeFsRoot, err) + } + + // create fakeFsRoot dir + if err = os.MkdirAll(fakeFsRoot, os.FileMode(0755)); err != nil { + return nil, nil, fmt.Errorf("failed to create fake root dir: %s. %s", fakeFsRoot, err) + } + + return &FakeFs{a: afero.Afero{Fs: afero.NewBasePathFs(afero.NewOsFs(), fakeFsRoot)}}, + func() { + os.RemoveAll(fakeFsRoot) + }, + nil +} + +// Stat via afero.Fs.Stat +func (fs *FakeFs) Stat(name string) (os.FileInfo, error) { + return fs.a.Fs.Stat(name) +} + +// Create via afero.Fs.Create +func (fs *FakeFs) Create(name string) (File, error) { + file, err := fs.a.Fs.Create(name) + if err != nil { + return nil, err + } + return &fakeFile{file}, nil +} + +// Rename via afero.Fs.Rename +func (fs *FakeFs) Rename(oldpath, newpath string) error { + return fs.a.Fs.Rename(oldpath, newpath) +} + +// MkdirAll via afero.Fs.MkdirAll +func (fs *FakeFs) MkdirAll(path string, perm os.FileMode) error { + return fs.a.Fs.MkdirAll(path, perm) +} + +// Chtimes via afero.Fs.Chtimes +func (fs *FakeFs) Chtimes(name string, atime, mtime time.Time) error { + return fs.a.Fs.Chtimes(name, atime, mtime) +} + +// ReadFile via afero.ReadFile +func (fs *FakeFs) ReadFile(filename string) ([]byte, error) { + return fs.a.ReadFile(filename) +} + +// WriteFile via afero.WriteFile +func (fs *FakeFs) WriteFile(filename string, data []byte, perm os.FileMode) error { + return fs.a.WriteFile(filename, data, perm) +} + +// TempDir via afero.TempDir +func (fs *FakeFs) TempDir(dir, prefix string) (string, error) { + return fs.a.TempDir(dir, prefix) +} + +// TempFile via afero.TempFile +func (fs *FakeFs) TempFile(dir, prefix string) (File, error) { + file, err := fs.a.TempFile(dir, prefix) + if err != nil { + return nil, err + } + return &fakeFile{file}, nil +} + +// ReadDir via afero.ReadDir +func (fs *FakeFs) ReadDir(dirname string) ([]os.FileInfo, error) { + return fs.a.ReadDir(dirname) +} + +// Walk via afero.Walk +func (fs *FakeFs) Walk(root string, walkFn filepath.WalkFunc) error { + return fs.a.Walk(root, walkFn) +} + +// RemoveAll via afero.RemoveAll +func (fs *FakeFs) RemoveAll(path string) error { + return fs.a.RemoveAll(path) +} + +// Remove via afero.Remove +func (fs *FakeFs) Remove(name string) error { + return fs.a.Remove(name) +} + +// Readlink via afero.ReadlinkIfPossible +func (fs *FakeFs) Readlink(name string) (string, error) { + return fs.a.Fs.(afero.Symlinker).ReadlinkIfPossible(name) +} + +// Symlink via afero.FS.(Symlinker).SymlinkIfPossible +func (fs *FakeFs) Symlink(oldname, newname string) error { + return fs.a.Fs.(afero.Symlinker).SymlinkIfPossible(oldname, newname) +} + +// fakeFile implements File; for use with FakeFs +type fakeFile struct { + file afero.File +} + +// Name via afero.File.Name +func (file *fakeFile) Name() string { + return file.file.Name() +} + +// Write via afero.File.Write +func (file *fakeFile) Write(b []byte) (n int, err error) { + return file.file.Write(b) +} + +// Sync via afero.File.Sync +func (file *fakeFile) Sync() error { + return file.file.Sync() +} + +// Close via afero.File.Close +func (file *fakeFile) Close() error { + return file.file.Close() +} diff --git a/vendor/github.com/k8snetworkplumbingwg/sriovnet/pkg/utils/filesystem/filesystem.go b/vendor/github.com/k8snetworkplumbingwg/sriovnet/pkg/utils/filesystem/filesystem.go new file mode 100644 index 000000000..99073b3cc --- /dev/null +++ b/vendor/github.com/k8snetworkplumbingwg/sriovnet/pkg/utils/filesystem/filesystem.go @@ -0,0 +1,41 @@ +package filesystem + +import ( + "os" + "path/filepath" + "time" +) + +var Fs Filesystem = DefaultFs{} + +// Filesystem is an interface that we can use to mock various filesystem operations +type Filesystem interface { + // from "os" + Stat(name string) (os.FileInfo, error) + Create(name string) (File, error) + Rename(oldpath, newpath string) error + MkdirAll(path string, perm os.FileMode) error + Chtimes(name string, atime time.Time, mtime time.Time) error + RemoveAll(path string) error + Remove(name string) error + Readlink(name string) (string, error) + Symlink(oldname, newname string) error + + // from "io/ioutil" + ReadFile(filename string) ([]byte, error) + WriteFile(filename string, data []byte, perm os.FileMode) error + TempDir(dir, prefix string) (string, error) + TempFile(dir, prefix string) (File, error) + ReadDir(dirname string) ([]os.FileInfo, error) + Walk(root string, walkFn filepath.WalkFunc) error +} + +// File is an interface that we can use to mock various filesystem operations typically +// accessed through the File object from the "os" package +type File interface { + // for now, the only os.File methods used are those below, add more as necessary + Name() string + Write(b []byte) (n int, err error) + Sync() error + Close() error +} diff --git a/vendor/github.com/k8snetworkplumbingwg/sriovnet/pkg/utils/netlinkops/netlinkops.go b/vendor/github.com/k8snetworkplumbingwg/sriovnet/pkg/utils/netlinkops/netlinkops.go new file mode 100644 index 000000000..ce458a315 --- /dev/null +++ b/vendor/github.com/k8snetworkplumbingwg/sriovnet/pkg/utils/netlinkops/netlinkops.go @@ -0,0 +1,114 @@ +package netlinkops + +import ( + "fmt" + "net" + + "github.com/vishvananda/netlink" +) + +var nlOpsImpl NetlinkOps + +// NetlinkOps is an interface wrapping netlink to be used by sriovnet +type NetlinkOps interface { + // LinkByName gets link by netdev name + LinkByName(name string) (netlink.Link, error) + // LinkSetUp sets Link state to up + LinkSetUp(link netlink.Link) error + // LinkSetVfHardwareAddr sets VF hardware address + LinkSetVfHardwareAddr(link netlink.Link, vf int, hwaddr net.HardwareAddr) error + // LinkSetVfVlan sets VF vlan + LinkSetVfVlan(link netlink.Link, vf, vlan int) error + // LinkSetVfNodeGUID sets VF Node GUID + LinkSetVfNodeGUID(link netlink.Link, vf int, nodeguid net.HardwareAddr) error + // LinkSetVfPortGUID sets VF Port GUID + LinkSetVfPortGUID(link netlink.Link, vf int, portguid net.HardwareAddr) error + // LinkSetVfTrust sets VF trust for the given VF + LinkSetVfTrust(link netlink.Link, vf int, state bool) error + // LinkSetVfSpoofchk sets VF spoofchk for the given VF + LinkSetVfSpoofchk(link netlink.Link, vf int, check bool) error + // DevLinkGetAllPortList gets all devlink ports + DevLinkGetAllPortList() ([]*netlink.DevlinkPort, error) + // DevLinkGetPortByNetdevName gets devlink port by netdev name + DevLinkGetPortByNetdevName(netdev string) (*netlink.DevlinkPort, error) +} + +// GetNetlinkOps returns NetlinkOps interface +func GetNetlinkOps() NetlinkOps { + if nlOpsImpl == nil { + nlOpsImpl = &netlinkOps{} + } + return nlOpsImpl +} + +// SetNetlinkOps sets NetlinkOps interface (to be used by unit tests) +func SetNetlinkOps(nlops NetlinkOps) { + nlOpsImpl = nlops +} + +// ResetNetlinkOps resets nlOpsImpl to nil +func ResetNetlinkOps() { + nlOpsImpl = nil +} + +type netlinkOps struct{} + +// LinkByName gets link by netdev name +func (nlo *netlinkOps) LinkByName(name string) (netlink.Link, error) { + return netlink.LinkByName(name) +} + +// LinkSetUp sets Link state to up +func (nlo *netlinkOps) LinkSetUp(link netlink.Link) error { + return netlink.LinkSetUp(link) +} + +// LinkSetVfHardwareAddr sets VF hardware address +func (nlo *netlinkOps) LinkSetVfHardwareAddr(link netlink.Link, vf int, hwaddr net.HardwareAddr) error { + return netlink.LinkSetVfHardwareAddr(link, vf, hwaddr) +} + +// LinkSetVfVlan sets VF vlan +func (nlo *netlinkOps) LinkSetVfVlan(link netlink.Link, vf, vlan int) error { + return netlink.LinkSetVfVlan(link, vf, vlan) +} + +// LinkSetVfNodeGUID sets VF Node GUID +func (nlo *netlinkOps) LinkSetVfNodeGUID(link netlink.Link, vf int, nodeguid net.HardwareAddr) error { + return netlink.LinkSetVfNodeGUID(link, vf, nodeguid) +} + +// LinkSetVfPortGUID sets VF Port GUID +func (nlo *netlinkOps) LinkSetVfPortGUID(link netlink.Link, vf int, portguid net.HardwareAddr) error { + return netlink.LinkSetVfPortGUID(link, vf, portguid) +} + +// LinkSetVfTrust sets VF trust for the given VF +func (nlo *netlinkOps) LinkSetVfTrust(link netlink.Link, vf int, state bool) error { + return netlink.LinkSetVfTrust(link, vf, state) +} + +// LinkSetVfSpoofchk sets VF spoofchk for the given VF +func (nlo *netlinkOps) LinkSetVfSpoofchk(link netlink.Link, vf int, check bool) error { + return netlink.LinkSetVfSpoofchk(link, vf, check) +} + +// DevLinkGetAllPortList gets all devlink ports +func (nlo *netlinkOps) DevLinkGetAllPortList() ([]*netlink.DevlinkPort, error) { + return netlink.DevLinkGetAllPortList() +} + +// DevLinkGetPortByNetdevName gets devlink port by netdev name +func (nlo *netlinkOps) DevLinkGetPortByNetdevName(netdev string) (*netlink.DevlinkPort, error) { + ports, err := netlink.DevLinkGetAllPortList() + if err != nil { + return nil, err + } + + for _, port := range ports { + if netdev == port.NetdeviceName { + return port, nil + } + } + return nil, fmt.Errorf("failed to get devlink port for netdev %s", netdev) +} diff --git a/vendor/github.com/k8snetworkplumbingwg/sriovnet/sriovnet.go b/vendor/github.com/k8snetworkplumbingwg/sriovnet/sriovnet.go new file mode 100644 index 000000000..09dfacfe0 --- /dev/null +++ b/vendor/github.com/k8snetworkplumbingwg/sriovnet/sriovnet.go @@ -0,0 +1,488 @@ +package sriovnet + +import ( + "fmt" + "log" + "net" + "os" + "path" + "path/filepath" + "regexp" + "strconv" + "strings" + + "github.com/google/uuid" + "github.com/vishvananda/netlink" + + utilfs "github.com/k8snetworkplumbingwg/sriovnet/pkg/utils/filesystem" + "github.com/k8snetworkplumbingwg/sriovnet/pkg/utils/netlinkops" +) + +const ( + // Used locally + etherEncapType = "ether" + ibEncapType = "infiniband" +) + +var ( + virtFnRe = regexp.MustCompile(`virtfn(\d+)`) + pciAddressRe = regexp.MustCompile(`^[0-9a-f]{4}:[0-9a-f]{2}:[01][0-9a-f].[0-7]$`) + auxiliaryDeviceRe = regexp.MustCompile(`^(\S+\.){2}\d+$`) +) + +type VfObj struct { + Index int + PciAddress string + Bound bool + Allocated bool +} + +type PfNetdevHandle struct { + PfNetdevName string + pfLinkHandle netlink.Link + + List []*VfObj +} + +func SetPFLinkUp(pfNetdevName string) error { + handle, err := netlinkops.GetNetlinkOps().LinkByName(pfNetdevName) + if err != nil { + return err + } + + return netlinkops.GetNetlinkOps().LinkSetUp(handle) +} + +func IsVfPciVfioBound(pciAddr string) bool { + driverLink := filepath.Join(PciSysDir, pciAddr, "driver") + driverPath, err := utilfs.Fs.Readlink(driverLink) + if err != nil { + return false + } + driverName := filepath.Base(driverPath) + return driverName == "vfio-pci" +} + +func IsSriovSupported(netdevName string) bool { + maxvfs, err := getMaxVfCount(netdevName) + if maxvfs == 0 || err != nil { + return false + } + return true +} + +func IsSriovEnabled(netdevName string) bool { + curvfs, err := getCurrentVfCount(netdevName) + if curvfs == 0 || err != nil { + return false + } + return true +} + +func EnableSriov(pfNetdevName string) error { + var maxVfCount int + var err error + + devDirName := netDevDeviceDir(pfNetdevName) + + devExist := dirExists(devDirName) + if !devExist { + return fmt.Errorf("device %s not found", pfNetdevName) + } + + maxVfCount, err = getMaxVfCount(pfNetdevName) + if err != nil { + log.Println("Fail to read max vf count of PF", pfNetdevName) + return err + } + + if maxVfCount == 0 { + return fmt.Errorf("sriov unsupported for device: %s", pfNetdevName) + } + + curVfCount, err2 := getCurrentVfCount(pfNetdevName) + if err2 != nil { + log.Println("Fail to read current vf count of PF", pfNetdevName) + return err + } + if curVfCount == 0 { + return setMaxVfCount(pfNetdevName, maxVfCount) + } + return nil +} + +func DisableSriov(pfNetdevName string) error { + devDirName := netDevDeviceDir(pfNetdevName) + + devExist := dirExists(devDirName) + if !devExist { + return fmt.Errorf("device %s not found", pfNetdevName) + } + + return setMaxVfCount(pfNetdevName, 0) +} + +func GetPfNetdevHandle(pfNetdevName string) (*PfNetdevHandle, error) { + pfLinkHandle, err := netlinkops.GetNetlinkOps().LinkByName(pfNetdevName) + if err != nil { + return nil, err + } + + handle := PfNetdevHandle{ + PfNetdevName: pfNetdevName, + pfLinkHandle: pfLinkHandle, + } + + list, err := GetVfPciDevList(pfNetdevName) + if err != nil { + return nil, err + } + + for _, vfDir := range list { + vfIndexStr := strings.TrimPrefix(vfDir, netDevVfDevicePrefix) + vfIndex, _ := strconv.Atoi(vfIndexStr) + vfNetdevName := vfNetdevNameFromParent(pfNetdevName, vfIndex) + pciAddress, err := vfPCIDevNameFromVfIndex(pfNetdevName, vfIndex) + if err != nil { + log.Printf("Failed to read PCI Address for VF %v from PF %v: %v\n", + vfNetdevName, pfNetdevName, err) + continue + } + vfObj := VfObj{ + Index: vfIndex, + PciAddress: pciAddress, + } + if vfNetdevName != "" { + vfObj.Bound = true + } else { + vfObj.Bound = false + } + vfObj.Allocated = false + handle.List = append(handle.List, &vfObj) + } + return &handle, nil +} + +func UnbindVf(handle *PfNetdevHandle, vf *VfObj) error { + cmdFile := filepath.Join(NetSysDir, handle.PfNetdevName, netdevDriverDir, netdevUnbindFile) + cmdFileObj := fileObject{ + Path: cmdFile, + } + err := cmdFileObj.Write(vf.PciAddress) + if err != nil { + vf.Bound = false + } + return err +} + +func BindVf(handle *PfNetdevHandle, vf *VfObj) error { + cmdFile := filepath.Join(NetSysDir, handle.PfNetdevName, netdevDriverDir, netdevBindFile) + cmdFileObj := fileObject{ + Path: cmdFile, + } + err := cmdFileObj.Write(vf.PciAddress) + if err != nil { + vf.Bound = true + } + return err +} + +func GetVfDefaultMacAddr(vfNetdevName string) (string, error) { + ethHandle, err1 := netlinkops.GetNetlinkOps().LinkByName(vfNetdevName) + if err1 != nil { + return "", err1 + } + + ethAttr := ethHandle.Attrs() + return ethAttr.HardwareAddr.String(), nil +} + +func SetVfDefaultMacAddress(handle *PfNetdevHandle, vf *VfObj) error { + netdevName := vfNetdevNameFromParent(handle.PfNetdevName, vf.Index) + ethHandle, err1 := netlinkops.GetNetlinkOps().LinkByName(netdevName) + if err1 != nil { + return err1 + } + ethAttr := ethHandle.Attrs() + return netlinkops.GetNetlinkOps().LinkSetVfHardwareAddr(handle.pfLinkHandle, vf.Index, ethAttr.HardwareAddr) +} + +func SetVfVlan(handle *PfNetdevHandle, vf *VfObj, vlan int) error { + return netlinkops.GetNetlinkOps().LinkSetVfVlan(handle.pfLinkHandle, vf.Index, vlan) +} + +func setVfNodeGUID(handle *PfNetdevHandle, vf *VfObj, guid []byte) error { + var err error + + nodeGUIDHwAddr := net.HardwareAddr(guid) + + err = ibSetNodeGUID(handle.PfNetdevName, vf.Index, nodeGUIDHwAddr) + if err == nil { + return nil + } + err = netlinkops.GetNetlinkOps().LinkSetVfNodeGUID(handle.pfLinkHandle, vf.Index, guid) + return err +} + +func setVfPortGUID(handle *PfNetdevHandle, vf *VfObj, guid []byte) error { + var err error + + portGUIDHwAddr := net.HardwareAddr(guid) + + err = ibSetPortGUID(handle.PfNetdevName, vf.Index, portGUIDHwAddr) + if err == nil { + return nil + } + err = netlinkops.GetNetlinkOps().LinkSetVfPortGUID(handle.pfLinkHandle, vf.Index, guid) + return err +} + +func SetVfDefaultGUID(handle *PfNetdevHandle, vf *VfObj) error { + randUUID, err := uuid.NewRandom() + if err != nil { + return err + } + guid := randUUID[0:8] + guid[7] = byte(vf.Index) + + err = setVfNodeGUID(handle, vf, guid) + if err != nil { + return err + } + + err = setVfPortGUID(handle, vf, guid) + return err +} + +func SetVfPrivileged(handle *PfNetdevHandle, vf *VfObj, privileged bool) error { + var spoofChk bool + var trusted bool + + ethAttr := handle.pfLinkHandle.Attrs() + if ethAttr.EncapType != etherEncapType { + return nil + } + // Only ether type is supported + if privileged { + spoofChk = false + trusted = true + } else { + spoofChk = true + trusted = false + } + + /* do not check for error status as older kernels doesn't + * have support for it. + * golangci-lint complains on missing error check. ignore it + * with nolint comment until we update the code to ignore ENOTSUP error + */ + netlinkops.GetNetlinkOps().LinkSetVfTrust(handle.pfLinkHandle, vf.Index, trusted) //nolint + netlinkops.GetNetlinkOps().LinkSetVfSpoofchk(handle.pfLinkHandle, vf.Index, spoofChk) //nolint + return nil +} + +func setDefaultHwAddr(handle *PfNetdevHandle, vf *VfObj) error { + var err error + + ethAttr := handle.pfLinkHandle.Attrs() + if ethAttr.EncapType == etherEncapType { + err = SetVfDefaultMacAddress(handle, vf) + } else if ethAttr.EncapType == ibEncapType { + err = SetVfDefaultGUID(handle, vf) + } + return err +} + +func setPortAdminState(handle *PfNetdevHandle, vf *VfObj) error { + ethAttr := handle.pfLinkHandle.Attrs() + if ethAttr.EncapType == ibEncapType { + state, err2 := ibGetPortAdminState(handle.PfNetdevName, vf.Index) + // Ignore the error where this file is not available + if err2 != nil { + return nil + } + log.Printf("Admin state = %v", state) + err2 = ibSetPortAdminState(handle.PfNetdevName, vf.Index, ibSriovPortAdminStateFollow) + if err2 != nil { + // If file exist, we must be able to write + log.Printf("Admin state setting error = %v", err2) + return err2 + } + } + return nil +} + +func ConfigVfs(handle *PfNetdevHandle, privileged bool) error { + var err error + + for _, vf := range handle.List { + log.Printf("vf = %v\n", vf) + err = setPortAdminState(handle, vf) + if err != nil { + break + } + // skip VFs in another namespace + netdevName := vfNetdevNameFromParent(handle.PfNetdevName, vf.Index) + if _, err = netlinkops.GetNetlinkOps().LinkByName(netdevName); err != nil { + continue + } + err = setDefaultHwAddr(handle, vf) + if err != nil { + break + } + _ = SetVfPrivileged(handle, vf, privileged) + } + if err != nil { + return err + } + for _, vf := range handle.List { + if !vf.Bound { + continue + } + + err = UnbindVf(handle, vf) + if err != nil { + log.Printf("Fail to unbind err=%v\n", err) + break + } + + err = BindVf(handle, vf) + if err != nil { + log.Printf("Fail to bind err=%v\n", err) + break + } + log.Printf("vf = %v unbind/bind completed", vf) + } + return nil +} + +func AllocateVf(handle *PfNetdevHandle) (*VfObj, error) { + for _, vf := range handle.List { + if vf.Allocated { + continue + } + vf.Allocated = true + log.Printf("Allocated vf = %v\n", *vf) + return vf, nil + } + return nil, fmt.Errorf("all Vfs for %v are allocated", handle.PfNetdevName) +} + +func AllocateVfByMacAddress(handle *PfNetdevHandle, vfMacAddress string) (*VfObj, error) { + for _, vf := range handle.List { + if vf.Allocated { + continue + } + + netdevName := vfNetdevNameFromParent(handle.PfNetdevName, vf.Index) + macAddr, _ := GetVfDefaultMacAddr(netdevName) + if macAddr != vfMacAddress { + continue + } + vf.Allocated = true + log.Printf("Allocated vf by mac = %v\n", *vf) + return vf, nil + } + return nil, fmt.Errorf("all Vfs for %v are allocated for mac address %v", + handle.PfNetdevName, vfMacAddress) +} + +func FreeVf(handle *PfNetdevHandle, vf *VfObj) { + vf.Allocated = false + log.Printf("Free vf = %v\n", *vf) +} + +func FreeVfByNetdevName(handle *PfNetdevHandle, vfIndex int) error { + vfNetdevName := fmt.Sprintf("%s%v", netDevVfDevicePrefix, vfIndex) + for _, vf := range handle.List { + netdevName := vfNetdevNameFromParent(handle.PfNetdevName, vf.Index) + if vf.Allocated && netdevName == vfNetdevName { + vf.Allocated = true + return nil + } + } + return fmt.Errorf("vf netdev %v not found", vfNetdevName) +} + +func GetVfNetdevName(handle *PfNetdevHandle, vf *VfObj) string { + return vfNetdevNameFromParent(handle.PfNetdevName, vf.Index) +} + +// GetVfIndexByPciAddress gets a VF PCI address (e.g '0000:03:00.4') and +// returns the correlate VF index. +func GetVfIndexByPciAddress(vfPciAddress string) (int, error) { + vfPath := filepath.Join(PciSysDir, vfPciAddress, "physfn", "virtfn*") + matches, err := filepath.Glob(vfPath) + if err != nil { + return -1, err + } + for _, match := range matches { + tmp, err := os.Readlink(match) + if err != nil { + continue + } + if strings.Contains(tmp, vfPciAddress) { + result := virtFnRe.FindStringSubmatch(match) + vfIndex, err := strconv.Atoi(result[1]) + if err != nil { + continue + } + return vfIndex, nil + } + } + return -1, fmt.Errorf("vf index for %s not found", vfPciAddress) +} + +// GetPfPciFromVfPci retrieves the parent PF PCI address of the provided VF PCI address in D:B:D.f format +func GetPfPciFromVfPci(vfPciAddress string) (string, error) { + pfPath := filepath.Join(PciSysDir, vfPciAddress, "physfn") + pciDevDir, err := utilfs.Fs.Readlink(pfPath) + if err != nil { + return "", fmt.Errorf("failed to read physfn link, provided address may not be a VF. %v", err) + } + + pf := path.Base(pciDevDir) + if pf == "" { + return pf, fmt.Errorf("could not find PF PCI Address") + } + return pf, err +} + +// GetNetDevicesFromPci gets a PCI address (e.g '0000:03:00.1') and +// returns the correlate list of netdevices +func GetNetDevicesFromPci(pciAddress string) ([]string, error) { + pciDir := filepath.Join(PciSysDir, pciAddress, "net") + return getFileNamesFromPath(pciDir) +} + +// GetPciFromNetDevice returns the PCI address associated with a network device name +func GetPciFromNetDevice(name string) (string, error) { + devPath := filepath.Join(NetSysDir, name) + + realPath, err := utilfs.Fs.Readlink(devPath) + if err != nil { + return "", fmt.Errorf("device %s not found: %s", name, err) + } + + parent := filepath.Dir(realPath) + base := filepath.Base(parent) + // Devices can have their PCI device sysfs entry at different levels: + // PF, VF, SF representor: + // /sys/devices/pci0000:00/.../0000:03:00.0/net/p0 + // /sys/devices/pci0000:00/.../0000:03:00.0/net/pf0hpf + // /sys/devices/pci0000:00/.../0000:03:00.0/net/pf0vf0 + // /sys/devices/pci0000:00/.../0000:03:00.0/net/pf0sf0 + // SF port: + // /sys/devices/pci0000:00/.../0000:03:00.0/mlx5_core.sf.3/net/enp3s0f0s1 + // This loop allows detecting any of them. + for parent != "/" && !pciAddressRe.MatchString(base) { + parent = filepath.Dir(parent) + base = filepath.Base(parent) + } + // If we stopped on '/' and the base was never a proper PCI address, + // then 'netdev' is not a PCI device. + if !pciAddressRe.MatchString(base) { + return "", fmt.Errorf("device %s is not a PCI device: %s", name, realPath) + } + return base, nil +} diff --git a/vendor/github.com/k8snetworkplumbingwg/sriovnet/sriovnet_aux.go b/vendor/github.com/k8snetworkplumbingwg/sriovnet/sriovnet_aux.go new file mode 100644 index 000000000..a60061b3e --- /dev/null +++ b/vendor/github.com/k8snetworkplumbingwg/sriovnet/sriovnet_aux.go @@ -0,0 +1,111 @@ +/*---------------------------------------------------- + * + * 2022 NVIDIA CORPORATION & AFFILIATES + * + * Licensed under the Apache License, Version 2.0 (the License); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + *---------------------------------------------------- + */ + +package sriovnet + +import ( + "fmt" + "path/filepath" + "strconv" + "strings" + + utilfs "github.com/k8snetworkplumbingwg/sriovnet/pkg/utils/filesystem" +) + +// GetNetDeviceFromAux gets auxiliary device name (e.g 'mlx5_core.sf.2') and +// returns the correlate netdevice +func GetNetDevicesFromAux(auxDev string) ([]string, error) { + auxDir := filepath.Join(AuxSysDir, auxDev, "net") + return getFileNamesFromPath(auxDir) +} + +// GetSfIndexByAuxDev gets a SF device name (e.g 'mlx5_core.sf.2') and +// returns the correlate SF index. +func GetSfIndexByAuxDev(auxDev string) (int, error) { + sfNumFile := filepath.Join(AuxSysDir, auxDev, "sfnum") + if _, err := utilfs.Fs.Stat(sfNumFile); err != nil { + return -1, fmt.Errorf("cannot get sfnum for %s device: %v", auxDev, err) + } + + sfNumStr, err := utilfs.Fs.ReadFile(sfNumFile) + if err != nil { + return -1, fmt.Errorf("cannot read sfnum file for %s device: %v", auxDev, err) + } + + sfnum, err := strconv.Atoi(strings.TrimSpace(string(sfNumStr))) + if err != nil { + return -1, err + } + return sfnum, nil +} + +// GetPfPciFromAux retrieves the parent PF PCI address of the provided auxiliary device in D.T.f format +func GetPfPciFromAux(auxDev string) (string, error) { + auxPath := filepath.Join(AuxSysDir, auxDev) + absoluteAuxPath, err := utilfs.Fs.Readlink(auxPath) + if err != nil { + return "", fmt.Errorf("failed to read auxiliary link, provided device ID may be not auxiliary device. %v", err) + } + // /sys/bus/auxiliary/devices/mlx5_core.sf.7 -> + // ./../../devices/pci0000:00/0000:00:00.0/0000:01:00.0/0000:02:00.0/0000:03:00.0/mlx5_core.sf.7 + parent := filepath.Dir(absoluteAuxPath) + base := filepath.Base(parent) + for !pciAddressRe.MatchString(base) { + // it's a nested auxiliary device. repeat + parent = filepath.Dir(parent) + base = filepath.Base(parent) + } + if base == "" { + return base, fmt.Errorf("could not find PF PCI Address") + } + return base, err +} + +// GetUplinkRepresentorFromAux gets auxiliary device name (e.g 'mlx5_core.sf.2') and +// returns the uplink representor netdev name for device. +func GetUplinkRepresentorFromAux(auxDev string) (string, error) { + pfPci, err := GetPfPciFromAux(auxDev) + if err != nil { + return "", fmt.Errorf("failed to find uplink PCI device: %v", err) + } + + return GetUplinkRepresentor(pfPci) +} + +// GetAuxNetDevicesFromPci returns a list of auxiliary devices names for the specified PCI network device +func GetAuxNetDevicesFromPci(pciAddr string) ([]string, error) { + baseDev := filepath.Join(PciSysDir, pciAddr) + // ensure that "net" folder exists, meaning it is network PCI device + if _, err := utilfs.Fs.Stat(filepath.Join(baseDev, "net")); err != nil { + return nil, err + } + + files, err := utilfs.Fs.ReadDir(baseDev) + if err != nil { + return nil, err + } + + auxDevs := make([]string, 0) + for _, file := range files { + if auxiliaryDeviceRe.MatchString(file.Name()) { + auxDevs = append(auxDevs, file.Name()) + } + } + return auxDevs, nil +} diff --git a/vendor/github.com/k8snetworkplumbingwg/sriovnet/sriovnet_helper.go b/vendor/github.com/k8snetworkplumbingwg/sriovnet/sriovnet_helper.go new file mode 100644 index 000000000..46ab4fb7e --- /dev/null +++ b/vendor/github.com/k8snetworkplumbingwg/sriovnet/sriovnet_helper.go @@ -0,0 +1,130 @@ +package sriovnet + +import ( + "fmt" + "log" + "os" + "path/filepath" +) + +const ( + NetSysDir = "/sys/class/net" + PciSysDir = "/sys/bus/pci/devices" + AuxSysDir = "/sys/bus/auxiliary/devices" + pcidevPrefix = "device" + netdevDriverDir = "device/driver" + netdevUnbindFile = "unbind" + netdevBindFile = "bind" + + netDevMaxVfCountFile = "sriov_totalvfs" + netDevCurrentVfCountFile = "sriov_numvfs" + netDevVfDevicePrefix = "virtfn" +) + +type VfObject struct { + NetdevName string + PCIDevName string +} + +func netDevDeviceDir(netDevName string) string { + devDirName := filepath.Join(NetSysDir, netDevName, pcidevPrefix) + return devDirName +} + +func getMaxVfCount(pfNetdevName string) (int, error) { + devDirName := netDevDeviceDir(pfNetdevName) + + maxDevFile := fileObject{ + Path: filepath.Join(devDirName, netDevMaxVfCountFile), + } + + maxVfs, err := maxDevFile.ReadInt() + if err != nil { + return 0, err + } + log.Println("max_vfs = ", maxVfs) + return maxVfs, nil +} + +func setMaxVfCount(pfNetdevName string, maxVfs int) error { + devDirName := netDevDeviceDir(pfNetdevName) + + maxDevFile := fileObject{ + Path: filepath.Join(devDirName, netDevCurrentVfCountFile), + } + + return maxDevFile.WriteInt(maxVfs) +} + +func getCurrentVfCount(pfNetdevName string) (int, error) { + devDirName := netDevDeviceDir(pfNetdevName) + + maxDevFile := fileObject{ + Path: filepath.Join(devDirName, netDevCurrentVfCountFile), + } + + curVfs, err := maxDevFile.ReadInt() + if err != nil { + return 0, err + } + log.Println("cur_vfs = ", curVfs) + return curVfs, nil +} + +func vfNetdevNameFromParent(pfNetdevName string, vfIndex int) string { + devDirName := netDevDeviceDir(pfNetdevName) + vfNetdev, _ := lsFilesWithPrefix(fmt.Sprintf("%s/%s%v/net", devDirName, + netDevVfDevicePrefix, vfIndex), "", false) + if len(vfNetdev) == 0 { + return "" + } + return vfNetdev[0] +} + +func readPCIsymbolicLink(symbolicLink string) (string, error) { + pciDevDir, err := os.Readlink(symbolicLink) + //nolint:gomnd + if len(pciDevDir) <= 3 { + return "", fmt.Errorf("could not find PCI Address") + } + + return pciDevDir[3:], err +} +func vfPCIDevNameFromVfIndex(pfNetdevName string, vfIndex int) (string, error) { + symbolicLink := filepath.Join(NetSysDir, pfNetdevName, pcidevPrefix, fmt.Sprintf("%s%v", + netDevVfDevicePrefix, vfIndex)) + pciAddress, err := readPCIsymbolicLink(symbolicLink) + if err != nil { + err = fmt.Errorf("%v for VF %s%v of PF %s", err, + netDevVfDevicePrefix, vfIndex, pfNetdevName) + } + return pciAddress, err +} + +func getPCIFromDeviceName(netdevName string) (string, error) { + symbolicLink := filepath.Join(NetSysDir, netdevName, pcidevPrefix) + pciAddress, err := readPCIsymbolicLink(symbolicLink) + if err != nil { + err = fmt.Errorf("%v for netdevice %s", err, netdevName) + } + return pciAddress, err +} + +func GetVfPciDevList(pfNetdevName string) ([]string, error) { + var i int + devDirName := netDevDeviceDir(pfNetdevName) + + virtFnDirs, err := lsFilesWithPrefix(devDirName, netDevVfDevicePrefix, true) + + if err != nil { + return nil, err + } + + i = 0 + vfDirList := make([]string, 0, len(virtFnDirs)) + for _, vfDir := range virtFnDirs { + vfDirList = append(vfDirList, vfDir) + i++ + } + return vfDirList, nil +} diff --git a/vendor/github.com/k8snetworkplumbingwg/sriovnet/sriovnet_switchdev.go b/vendor/github.com/k8snetworkplumbingwg/sriovnet/sriovnet_switchdev.go new file mode 100644 index 000000000..5ccf3fadc --- /dev/null +++ b/vendor/github.com/k8snetworkplumbingwg/sriovnet/sriovnet_switchdev.go @@ -0,0 +1,499 @@ +package sriovnet + +import ( + "bytes" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + + utilfs "github.com/k8snetworkplumbingwg/sriovnet/pkg/utils/filesystem" + "github.com/k8snetworkplumbingwg/sriovnet/pkg/utils/netlinkops" +) + +const ( + netdevPhysSwitchID = "phys_switch_id" + netdevPhysPortName = "phys_port_name" +) + +type PortFlavour uint16 + +// Keep things consistent with netlink lib constants +// nolint:revive,stylecheck +const ( + PORT_FLAVOUR_PHYSICAL = iota + PORT_FLAVOUR_CPU + PORT_FLAVOUR_DSA + PORT_FLAVOUR_PCI_PF + PORT_FLAVOUR_PCI_VF + PORT_FLAVOUR_VIRTUAL + PORT_FLAVOUR_UNUSED + PORT_FLAVOUR_PCI_SF + PORT_FLAVOUR_UNKNOWN = 0xffff +) + +// Regex that matches on the physical/upling port name +var physPortRepRegex = regexp.MustCompile(`^p(\d+)$`) + +// Regex that matches on PF representor port name. These ports exists on DPUs. +var pfPortRepRegex = regexp.MustCompile(`^(?:c\d+)?pf(\d+)$`) + +// Regex that matches on VF representor port name +var vfPortRepRegex = regexp.MustCompile(`^(?:c\d+)?pf(\d+)vf(\d+)$`) + +// Regex that matches on SF representor port name +var sfPortRepRegex = regexp.MustCompile(`^(?:c\d+)?pf(\d+)sf(\d+)$`) + +func parseIndexFromPhysPortName(portName string, regex *regexp.Regexp) (pfRepIndex, vfRepIndex int, err error) { + pfRepIndex = -1 + vfRepIndex = -1 + + matches := regex.FindStringSubmatch(portName) + //nolint:gomnd + if len(matches) != 3 { + err = fmt.Errorf("failed to parse portName %s", portName) + } else { + pfRepIndex, err = strconv.Atoi(matches[1]) + if err == nil { + vfRepIndex, err = strconv.Atoi(matches[2]) + } + } + return pfRepIndex, vfRepIndex, err +} + +func parsePortName(physPortName string) (pfRepIndex, vfRepIndex int, err error) { + // old kernel syntax of phys_port_name is vf index + physPortName = strings.TrimSpace(physPortName) + physPortNameInt, err := strconv.Atoi(physPortName) + if err == nil { + vfRepIndex = physPortNameInt + } else { + pfRepIndex, vfRepIndex, err = parseIndexFromPhysPortName(physPortName, vfPortRepRegex) + } + return pfRepIndex, vfRepIndex, err +} + +func sfIndexFromPortName(physPortName string) (int, error) { + //nolint:gomnd + _, sfRepIndex, err := parseIndexFromPhysPortName(physPortName, sfPortRepRegex) + return sfRepIndex, err +} + +func isSwitchdev(netdevice string) bool { + swIDFile := filepath.Join(NetSysDir, netdevice, netdevPhysSwitchID) + physSwitchID, err := utilfs.Fs.ReadFile(swIDFile) + if err != nil { + return false + } + if len(physSwitchID) != 0 { + return true + } + return false +} + +// GetUplinkRepresentor gets a VF or PF PCI address (e.g '0000:03:00.4') and +// returns the uplink represntor netdev name for that VF or PF. +func GetUplinkRepresentor(pciAddress string) (string, error) { + devicePath := filepath.Join(PciSysDir, pciAddress, "physfn", "net") + if _, err := utilfs.Fs.Stat(devicePath); errors.Is(err, os.ErrNotExist) { + // If physfn symlink to the parent PF doesn't exist, use the current device's dir + devicePath = filepath.Join(PciSysDir, pciAddress, "net") + } + + devices, err := utilfs.Fs.ReadDir(devicePath) + if err != nil { + return "", fmt.Errorf("failed to lookup %s: %v", pciAddress, err) + } + for _, device := range devices { + if isSwitchdev(device.Name()) { + // Try to get the phys port name, if not exists then fallback to check without it + // phys_port_name should be in formant p e.g p0,p1,p2 ...etc. + if devicePhysPortName, err := getNetDevPhysPortName(device.Name()); err == nil { + if !physPortRepRegex.MatchString(devicePhysPortName) { + continue + } + } + + return device.Name(), nil + } + } + return "", fmt.Errorf("uplink for %s not found", pciAddress) +} + +func GetVfRepresentor(uplink string, vfIndex int) (string, error) { + swIDFile := filepath.Join(NetSysDir, uplink, netdevPhysSwitchID) + physSwitchID, err := utilfs.Fs.ReadFile(swIDFile) + if err != nil || len(physSwitchID) == 0 { + return "", fmt.Errorf("cant get uplink %s switch id", uplink) + } + + pfSubsystemPath := filepath.Join(NetSysDir, uplink, "subsystem") + devices, err := utilfs.Fs.ReadDir(pfSubsystemPath) + if err != nil { + return "", err + } + for _, device := range devices { + devicePath := filepath.Join(NetSysDir, device.Name()) + deviceSwIDFile := filepath.Join(devicePath, netdevPhysSwitchID) + deviceSwID, err := utilfs.Fs.ReadFile(deviceSwIDFile) + if err != nil || !bytes.Equal(deviceSwID, physSwitchID) { + continue + } + physPortNameStr, err := getNetDevPhysPortName(device.Name()) + if err != nil { + continue + } + pfRepIndex, vfRepIndex, _ := parsePortName(physPortNameStr) + if pfRepIndex != -1 { + pfPCIAddress, err := getPCIFromDeviceName(uplink) + if err != nil { + continue + } + PCIFuncAddress, err := strconv.Atoi(string((pfPCIAddress[len(pfPCIAddress)-1]))) + if pfRepIndex != PCIFuncAddress || err != nil { + continue + } + } + // At this point we're confident we have a representor. + if vfRepIndex == vfIndex { + return device.Name(), nil + } + } + return "", fmt.Errorf("failed to find VF representor for uplink %s", uplink) +} + +func GetSfRepresentor(uplink string, sfNum int) (string, error) { + pfNetPath := filepath.Join(NetSysDir, uplink, "device", "net") + devices, err := utilfs.Fs.ReadDir(pfNetPath) + if err != nil { + return "", err + } + + for _, device := range devices { + physPortNameStr, err := getNetDevPhysPortName(device.Name()) + if err != nil { + continue + } + sfRepIndex, err := sfIndexFromPortName(physPortNameStr) + if err != nil { + continue + } + if sfRepIndex == sfNum { + return device.Name(), nil + } + } + return "", fmt.Errorf("failed to find SF representor for uplink %s", uplink) +} + +func getNetDevPhysPortName(netDev string) (string, error) { + devicePortNameFile := filepath.Join(NetSysDir, netDev, netdevPhysPortName) + physPortName, err := utilfs.Fs.ReadFile(devicePortNameFile) + if err != nil { + return "", err + } + return strings.TrimSpace(string(physPortName)), nil +} + +// findNetdevWithPortNameCriteria returns representor netdev that matches a criteria function on the +// physical port name +func findNetdevWithPortNameCriteria(criteria func(string) bool) (string, error) { + netdevs, err := utilfs.Fs.ReadDir(NetSysDir) + if err != nil { + return "", err + } + + for _, netdev := range netdevs { + // find matching VF representor + netdevName := netdev.Name() + + // skip non switchdev netdevs + if !isSwitchdev(netdevName) { + continue + } + + portName, err := getNetDevPhysPortName(netdevName) + if err != nil { + continue + } + + if criteria(portName) { + return netdevName, nil + } + } + return "", fmt.Errorf("no representor matched criteria") +} + +// GetPortIndexFromRepresentor finds the index of a representor from its network device name. +// Supports VF and SF. For multiple port flavors, the same ID could be returned, i.e. +// +// pf0vf10 and pf0sf10 +// +// will return the same port ID. To further differentiate the ports, use GetRepresentorPortFlavour +func GetPortIndexFromRepresentor(repNetDev string) (int, error) { + flavor, err := GetRepresentorPortFlavour(repNetDev) + if err != nil { + return 0, err + } + + if flavor != PORT_FLAVOUR_PCI_VF && flavor != PORT_FLAVOUR_PCI_SF { + return 0, fmt.Errorf("unsupported port flavor for netdev %s", repNetDev) + } + + physPortName, err := getNetDevPhysPortName(repNetDev) + if err != nil { + return 0, fmt.Errorf("failed to get device %s physical port name: %v", repNetDev, err) + } + + typeToRegex := map[PortFlavour]*regexp.Regexp{ + PORT_FLAVOUR_PCI_VF: vfPortRepRegex, + PORT_FLAVOUR_PCI_SF: sfPortRepRegex, + } + + _, repIndex, err := parseIndexFromPhysPortName(physPortName, typeToRegex[flavor]) + if err != nil { + return 0, fmt.Errorf("failed to parse the physical port name of device %s: %v", repNetDev, err) + } + + return repIndex, nil +} + +// GetVfRepresentorDPU returns VF representor on DPU for a host VF identified by pfID and vfIndex +func GetVfRepresentorDPU(pfID, vfIndex string) (string, error) { + // TODO(Adrianc): This method should change to get switchID and vfIndex as input, then common logic can + // be shared with GetVfRepresentor, backward compatibility should be preserved when this happens. + + // pfID should be 0 or 1 + if pfID != "0" && pfID != "1" { + return "", fmt.Errorf("unexpected pfID(%s). It should be 0 or 1", pfID) + } + + // vfIndex should be an unsinged integer provided as a decimal number + if _, err := strconv.ParseUint(vfIndex, 10, 32); err != nil { + return "", fmt.Errorf("unexpected vfIndex(%s). It should be an unsigned decimal number", vfIndex) + } + + // map for easy search of expected VF rep port name. + // Note: no support for Multi-Chassis DPUs + expectedPhysPortNames := map[string]interface{}{ + fmt.Sprintf("pf%svf%s", pfID, vfIndex): nil, + fmt.Sprintf("c1pf%svf%s", pfID, vfIndex): nil, + } + + netdev, err := findNetdevWithPortNameCriteria(func(portName string) bool { + // if phys port name == pfvf or c1pfvf we have a match + if _, ok := expectedPhysPortNames[portName]; ok { + return true + } + return false + }) + + if err != nil { + return "", fmt.Errorf("vf representor for pfID:%s, vfIndex:%s not found", pfID, vfIndex) + } + return netdev, nil +} + +// GetSfRepresentorDPU returns SF representor on DPU for a host SF identified by pfID and sfIndex +func GetSfRepresentorDPU(pfID, sfIndex string) (string, error) { + // pfID should be 0 or 1 + if pfID != "0" && pfID != "1" { + return "", fmt.Errorf("unexpected pfID(%s). It should be 0 or 1", pfID) + } + + // sfIndex should be an unsinged integer provided as a decimal number + if _, err := strconv.ParseUint(sfIndex, 10, 32); err != nil { + return "", fmt.Errorf("unexpected sfIndex(%s). It should be an unsigned decimal number", sfIndex) + } + + // map for easy search of expected VF rep port name. + // Note: no support for Multi-Chassis DPUs + expectedPhysPortNames := map[string]interface{}{ + fmt.Sprintf("pf%ssf%s", pfID, sfIndex): nil, + fmt.Sprintf("c1pf%ssf%s", pfID, sfIndex): nil, + } + + netdev, err := findNetdevWithPortNameCriteria(func(portName string) bool { + // if phys port name == pfsf or c1pfsf we have a match + if _, ok := expectedPhysPortNames[portName]; ok { + return true + } + return false + }) + + if err != nil { + return "", fmt.Errorf("sf representor for pfID:%s, sfIndex:%s not found", pfID, sfIndex) + } + return netdev, nil +} + +// GetRepresentorPortFlavour returns the representor port flavour +// Note: this method does not support old representor names used by old kernels +// e.g and will return PORT_FLAVOUR_UNKNOWN for such cases. +func GetRepresentorPortFlavour(netdev string) (PortFlavour, error) { + if !isSwitchdev(netdev) { + return PORT_FLAVOUR_UNKNOWN, fmt.Errorf("net device %s is does not represent an eswitch port", netdev) + } + + // Attempt to get information via devlink (Kernel >= 5.9.0) + port, err := netlinkops.GetNetlinkOps().DevLinkGetPortByNetdevName(netdev) + if err == nil { + return PortFlavour(port.PortFlavour), nil + } + + // Fallback to Get PortFlavour by phys_port_name + // read phy_port_name + portName, err := getNetDevPhysPortName(netdev) + if err != nil { + return PORT_FLAVOUR_UNKNOWN, err + } + + typeToRegex := map[PortFlavour]*regexp.Regexp{ + PORT_FLAVOUR_PHYSICAL: physPortRepRegex, + PORT_FLAVOUR_PCI_PF: pfPortRepRegex, + PORT_FLAVOUR_PCI_VF: vfPortRepRegex, + PORT_FLAVOUR_PCI_SF: sfPortRepRegex, + } + for flavour, regex := range typeToRegex { + if regex.MatchString(portName) { + return flavour, nil + } + } + return PORT_FLAVOUR_UNKNOWN, nil +} + +// parseDPUConfigFileOutput parses the config file content of a DPU +// representor port. The format of the file is a set of : pairs as follows: +// +// ``` +// +// MAC : 0c:42:a1:c6:cf:7c +// MaxTxRate : 0 +// State : Follow +// +// ``` +func parseDPUConfigFileOutput(out string) map[string]string { + configMap := make(map[string]string) + for _, line := range strings.Split(strings.TrimSuffix(out, "\n"), "\n") { + entry := strings.SplitN(line, ":", 2) + if len(entry) != 2 { + // unexpected line format + continue + } + configMap[strings.Trim(entry[0], " \t\n")] = strings.Trim(entry[1], " \t\n") + } + return configMap +} + +// GetRepresentorPeerMacAddress returns the MAC address of the peer netdev associated with the given +// representor netdev +// Note: +// +// This method functionality is currently supported only on DPUs. +// Currently only netdev representors with PORT_FLAVOUR_PCI_PF are supported +func GetRepresentorPeerMacAddress(netdev string) (net.HardwareAddr, error) { + flavor, err := GetRepresentorPortFlavour(netdev) + if err != nil { + return nil, fmt.Errorf("unknown port flavour for netdev %s. %v", netdev, err) + } + if flavor == PORT_FLAVOUR_UNKNOWN { + return nil, fmt.Errorf("unknown port flavour for netdev %s", netdev) + } + if flavor != PORT_FLAVOUR_PCI_PF { + return nil, fmt.Errorf("unsupported port flavour for netdev %s", netdev) + } + + // Attempt to get information via devlink (Kernel >= 5.9.0) + port, err := netlinkops.GetNetlinkOps().DevLinkGetPortByNetdevName(netdev) + if err == nil { + if port.Fn != nil { + return port.Fn.HwAddr, nil + } + } + + // Get information via sysfs + // read phy_port_name + portName, err := getNetDevPhysPortName(netdev) + if err != nil { + return nil, err + } + // Extract port num + portNum := pfPortRepRegex.FindStringSubmatch(portName) + if len(portNum) < 2 { + return nil, fmt.Errorf("failed to extract physical port number from port name %s of netdev %s", + portName, netdev) + } + uplinkPhysPortName := "p" + portNum[1] + // Find uplink netdev for that port + // Note(adrianc): As we support only DPUs ATM we do not need to deal with netdevs from different + // eswitch (i.e different switch IDs). + uplinkNetdev, err := findNetdevWithPortNameCriteria(func(pname string) bool { return pname == uplinkPhysPortName }) + if err != nil { + return nil, fmt.Errorf("failed to find uplink port for netdev %s. %v", netdev, err) + } + // get MAC address for netdev + configPath := filepath.Join(NetSysDir, uplinkNetdev, "smart_nic", "pf", "config") + out, err := utilfs.Fs.ReadFile(configPath) + if err != nil { + return nil, fmt.Errorf("failed to read DPU config via uplink %s for %s. %v", + uplinkNetdev, netdev, err) + } + config := parseDPUConfigFileOutput(string(out)) + macStr, ok := config["MAC"] + if !ok { + return nil, fmt.Errorf("MAC address not found for %s", netdev) + } + mac, err := net.ParseMAC(macStr) + if err != nil { + return nil, fmt.Errorf("failed to parse MAC address \"%s\" for %s. %v", macStr, netdev, err) + } + return mac, nil +} + +// SetRepresentorPeerMacAddress sets the given MAC addresss of the peer netdev associated with the given +// representor netdev. +// Note: This method functionality is currently supported only for DPUs. +// Currently only netdev representors with PORT_FLAVOUR_PCI_VF are supported +func SetRepresentorPeerMacAddress(netdev string, mac net.HardwareAddr) error { + flavor, err := GetRepresentorPortFlavour(netdev) + if err != nil { + return fmt.Errorf("unknown port flavour for netdev %s. %v", netdev, err) + } + if flavor == PORT_FLAVOUR_UNKNOWN { + return fmt.Errorf("unknown port flavour for netdev %s", netdev) + } + if flavor != PORT_FLAVOUR_PCI_VF { + return fmt.Errorf("unsupported port flavour for netdev %s", netdev) + } + + physPortNameStr, err := getNetDevPhysPortName(netdev) + if err != nil { + return fmt.Errorf("failed to get phys_port_name for netdev %s: %v", netdev, err) + } + pfID, vfIndex, err := parsePortName(physPortNameStr) + if err != nil { + return fmt.Errorf("failed to get the pf and vf index for netdev %s "+ + "with phys_port_name %s: %v", netdev, physPortNameStr, err) + } + + uplinkPhysPortName := fmt.Sprintf("p%d", pfID) + uplinkNetdev, err := findNetdevWithPortNameCriteria(func(pname string) bool { return pname == uplinkPhysPortName }) + if err != nil { + return fmt.Errorf("failed to find netdev for physical port name %s. %v", uplinkPhysPortName, err) + } + vfRepName := fmt.Sprintf("vf%d", vfIndex) + sysfsVfRepMacFile := filepath.Join(NetSysDir, uplinkNetdev, "smart_nic", vfRepName, "mac") + _, err = utilfs.Fs.Stat(sysfsVfRepMacFile) + if err != nil { + return fmt.Errorf("couldn't stat VF representor's sysfs file %s: %v", sysfsVfRepMacFile, err) + } + err = utilfs.Fs.WriteFile(sysfsVfRepMacFile, []byte(mac.String()), 0) + if err != nil { + return fmt.Errorf("failed to write the MAC address %s to VF reprentor %s", + mac.String(), sysfsVfRepMacFile) + } + return nil +} diff --git a/vendor/github.com/k8snetworkplumbingwg/sriovnet/utils.go b/vendor/github.com/k8snetworkplumbingwg/sriovnet/utils.go new file mode 100644 index 000000000..84772da95 --- /dev/null +++ b/vendor/github.com/k8snetworkplumbingwg/sriovnet/utils.go @@ -0,0 +1,45 @@ +/*---------------------------------------------------- + * + * 2022 NVIDIA CORPORATION & AFFILIATES + * + * Licensed under the Apache License, Version 2.0 (the License); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + *---------------------------------------------------- + */ + +package sriovnet + +import ( + "fmt" + "strings" + + utilfs "github.com/k8snetworkplumbingwg/sriovnet/pkg/utils/filesystem" +) + +func getFileNamesFromPath(dir string) ([]string, error) { + _, err := utilfs.Fs.Stat(dir) + if err != nil { + return nil, fmt.Errorf("could not stat the directory %s: %v", dir, err) + } + + files, err := utilfs.Fs.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("failed to read directory %s: %v", dir, err) + } + + netDevices := make([]string, 0, len(files)) + for _, netDeviceFile := range files { + netDevices = append(netDevices, strings.TrimSpace(netDeviceFile.Name())) + } + return netDevices, nil +} diff --git a/vendor/github.com/spf13/afero/afero.go b/vendor/github.com/spf13/afero/afero.go index 199480cd0..39f658520 100644 --- a/vendor/github.com/spf13/afero/afero.go +++ b/vendor/github.com/spf13/afero/afero.go @@ -97,7 +97,7 @@ type Fs interface { // Chown changes the uid and gid of the named file. Chown(name string, uid, gid int) error - //Chtimes changes the access and modification times of the named file + // Chtimes changes the access and modification times of the named file Chtimes(name string, atime time.Time, mtime time.Time) error } diff --git a/vendor/github.com/spf13/afero/basepath.go b/vendor/github.com/spf13/afero/basepath.go index 70a1d9168..2e72793a3 100644 --- a/vendor/github.com/spf13/afero/basepath.go +++ b/vendor/github.com/spf13/afero/basepath.go @@ -40,7 +40,6 @@ func (f *BasePathFile) Name() string { func (f *BasePathFile) ReadDir(n int) ([]fs.DirEntry, error) { if rdf, ok := f.File.(fs.ReadDirFile); ok { return rdf.ReadDir(n) - } return readDirFile{f.File}.ReadDir(n) } diff --git a/vendor/github.com/spf13/afero/copyOnWriteFs.go b/vendor/github.com/spf13/afero/copyOnWriteFs.go index 6ff8f3099..184d6dd70 100644 --- a/vendor/github.com/spf13/afero/copyOnWriteFs.go +++ b/vendor/github.com/spf13/afero/copyOnWriteFs.go @@ -223,7 +223,7 @@ func (u *CopyOnWriteFs) OpenFile(name string, flag int, perm os.FileMode) (File, return nil, err } if isaDir { - if err = u.layer.MkdirAll(dir, 0777); err != nil { + if err = u.layer.MkdirAll(dir, 0o777); err != nil { return nil, err } return u.layer.OpenFile(name, flag, perm) @@ -247,8 +247,9 @@ func (u *CopyOnWriteFs) OpenFile(name string, flag int, perm os.FileMode) (File, // This function handles the 9 different possibilities caused // by the union which are the intersection of the following... -// layer: doesn't exist, exists as a file, and exists as a directory -// base: doesn't exist, exists as a file, and exists as a directory +// +// layer: doesn't exist, exists as a file, and exists as a directory +// base: doesn't exist, exists as a file, and exists as a directory func (u *CopyOnWriteFs) Open(name string) (File, error) { // Since the overlay overrides the base we check that first b, err := u.isBaseFile(name) @@ -322,5 +323,5 @@ func (u *CopyOnWriteFs) MkdirAll(name string, perm os.FileMode) error { } func (u *CopyOnWriteFs) Create(name string) (File, error) { - return u.OpenFile(name, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0666) + return u.OpenFile(name, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0o666) } diff --git a/vendor/github.com/spf13/afero/ioutil.go b/vendor/github.com/spf13/afero/ioutil.go index 386c9cdc2..fa6abe1ee 100644 --- a/vendor/github.com/spf13/afero/ioutil.go +++ b/vendor/github.com/spf13/afero/ioutil.go @@ -141,8 +141,10 @@ func WriteFile(fs Fs, filename string, data []byte, perm os.FileMode) error { // We generate random temporary file names so that there's a good // chance the file doesn't exist yet - keeps the number of tries in // TempFile to a minimum. -var randNum uint32 -var randmu sync.Mutex +var ( + randNum uint32 + randmu sync.Mutex +) func reseed() uint32 { return uint32(time.Now().UnixNano() + int64(os.Getpid())) @@ -190,7 +192,7 @@ func TempFile(fs Fs, dir, pattern string) (f File, err error) { nconflict := 0 for i := 0; i < 10000; i++ { name := filepath.Join(dir, prefix+nextRandom()+suffix) - f, err = fs.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600) + f, err = fs.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o600) if os.IsExist(err) { if nconflict++; nconflict > 10 { randmu.Lock() @@ -214,6 +216,7 @@ func TempFile(fs Fs, dir, pattern string) (f File, err error) { func (a Afero) TempDir(dir, prefix string) (name string, err error) { return TempDir(a.Fs, dir, prefix) } + func TempDir(fs Fs, dir, prefix string) (name string, err error) { if dir == "" { dir = os.TempDir() @@ -222,7 +225,7 @@ func TempDir(fs Fs, dir, prefix string) (name string, err error) { nconflict := 0 for i := 0; i < 10000; i++ { try := filepath.Join(dir, prefix+nextRandom()) - err = fs.Mkdir(try, 0700) + err = fs.Mkdir(try, 0o700) if os.IsExist(err) { if nconflict++; nconflict > 10 { randmu.Lock() diff --git a/vendor/github.com/spf13/afero/mem/file.go b/vendor/github.com/spf13/afero/mem/file.go index 3cf4693b5..62fe4498e 100644 --- a/vendor/github.com/spf13/afero/mem/file.go +++ b/vendor/github.com/spf13/afero/mem/file.go @@ -245,7 +245,7 @@ func (f *File) Truncate(size int64) error { defer f.fileData.Unlock() if size > int64(len(f.fileData.data)) { diff := size - int64(len(f.fileData.data)) - f.fileData.data = append(f.fileData.data, bytes.Repeat([]byte{00}, int(diff))...) + f.fileData.data = append(f.fileData.data, bytes.Repeat([]byte{0o0}, int(diff))...) } else { f.fileData.data = f.fileData.data[0:size] } @@ -285,7 +285,7 @@ func (f *File) Write(b []byte) (n int, err error) { tail = f.fileData.data[n+int(cur):] } if diff > 0 { - f.fileData.data = append(f.fileData.data, append(bytes.Repeat([]byte{00}, int(diff)), b...)...) + f.fileData.data = append(f.fileData.data, append(bytes.Repeat([]byte{0o0}, int(diff)), b...)...) f.fileData.data = append(f.fileData.data, tail...) } else { f.fileData.data = append(f.fileData.data[:cur], b...) @@ -321,16 +321,19 @@ func (s *FileInfo) Name() string { s.Unlock() return name } + func (s *FileInfo) Mode() os.FileMode { s.Lock() defer s.Unlock() return s.mode } + func (s *FileInfo) ModTime() time.Time { s.Lock() defer s.Unlock() return s.modtime } + func (s *FileInfo) IsDir() bool { s.Lock() defer s.Unlock() diff --git a/vendor/github.com/spf13/afero/memmap.go b/vendor/github.com/spf13/afero/memmap.go index d06975e71..3f4ef42de 100644 --- a/vendor/github.com/spf13/afero/memmap.go +++ b/vendor/github.com/spf13/afero/memmap.go @@ -43,7 +43,7 @@ func (m *MemMapFs) getData() map[string]*mem.FileData { // Root should always exist, right? // TODO: what about windows? root := mem.CreateDir(FilePathSeparator) - mem.SetMode(root, os.ModeDir|0755) + mem.SetMode(root, os.ModeDir|0o755) m.data[FilePathSeparator] = root }) return m.data @@ -96,12 +96,12 @@ func (m *MemMapFs) registerWithParent(f *mem.FileData, perm os.FileMode) { pdir := filepath.Dir(filepath.Clean(f.Name())) err := m.lockfreeMkdir(pdir, perm) if err != nil { - //log.Println("Mkdir error:", err) + // log.Println("Mkdir error:", err) return } parent, err = m.lockfreeOpen(pdir) if err != nil { - //log.Println("Open after Mkdir error:", err) + // log.Println("Open after Mkdir error:", err) return } } @@ -319,6 +319,18 @@ func (m *MemMapFs) Rename(oldname, newname string) error { } else { return &os.PathError{Op: "rename", Path: oldname, Err: ErrFileNotFound} } + + for p, fileData := range m.getData() { + if strings.HasPrefix(p, oldname+FilePathSeparator) { + m.mu.RUnlock() + m.mu.Lock() + delete(m.getData(), p) + p := strings.Replace(p, oldname, newname, 1) + m.getData()[p] = fileData + m.mu.Unlock() + m.mu.RLock() + } + } return nil } diff --git a/vendor/github.com/spf13/afero/regexpfs.go b/vendor/github.com/spf13/afero/regexpfs.go index ac359c62a..218f3b235 100644 --- a/vendor/github.com/spf13/afero/regexpfs.go +++ b/vendor/github.com/spf13/afero/regexpfs.go @@ -10,7 +10,6 @@ import ( // The RegexpFs filters files (not directories) by regular expression. Only // files matching the given regexp will be allowed, all others get a ENOENT error ( // "No such file or directory"). -// type RegexpFs struct { re *regexp.Regexp source Fs diff --git a/vendor/github.com/spf13/afero/symlink.go b/vendor/github.com/spf13/afero/symlink.go index d1c6ea53d..aa6ae125b 100644 --- a/vendor/github.com/spf13/afero/symlink.go +++ b/vendor/github.com/spf13/afero/symlink.go @@ -21,9 +21,9 @@ import ( // filesystems saying so. // It indicates support for 3 symlink related interfaces that implement the // behaviors of the os methods: -// - Lstat -// - Symlink, and -// - Readlink +// - Lstat +// - Symlink, and +// - Readlink type Symlinker interface { Lstater Linker diff --git a/vendor/github.com/spf13/afero/unionFile.go b/vendor/github.com/spf13/afero/unionFile.go index 333d367f4..f02e75532 100644 --- a/vendor/github.com/spf13/afero/unionFile.go +++ b/vendor/github.com/spf13/afero/unionFile.go @@ -130,7 +130,7 @@ func (f *UnionFile) Name() string { type DirsMerger func(lofi, bofi []os.FileInfo) ([]os.FileInfo, error) var defaultUnionMergeDirsFn = func(lofi, bofi []os.FileInfo) ([]os.FileInfo, error) { - var files = make(map[string]os.FileInfo) + files := make(map[string]os.FileInfo) for _, fi := range lofi { files[fi.Name()] = fi @@ -151,7 +151,6 @@ var defaultUnionMergeDirsFn = func(lofi, bofi []os.FileInfo) ([]os.FileInfo, err } return rfi, nil - } // Readdir will weave the two directories together and @@ -275,7 +274,7 @@ func copyFile(base Fs, layer Fs, name string, bfh File) error { return err } if !exists { - err = layer.MkdirAll(filepath.Dir(name), 0777) // FIXME? + err = layer.MkdirAll(filepath.Dir(name), 0o777) // FIXME? if err != nil { return err } diff --git a/vendor/github.com/spf13/afero/util.go b/vendor/github.com/spf13/afero/util.go index cb7de23f2..9e4cba274 100644 --- a/vendor/github.com/spf13/afero/util.go +++ b/vendor/github.com/spf13/afero/util.go @@ -43,7 +43,7 @@ func WriteReader(fs Fs, path string, r io.Reader) (err error) { ospath := filepath.FromSlash(dir) if ospath != "" { - err = fs.MkdirAll(ospath, 0777) // rwx, rw, r + err = fs.MkdirAll(ospath, 0o777) // rwx, rw, r if err != nil { if err != os.ErrExist { return err @@ -71,7 +71,7 @@ func SafeWriteReader(fs Fs, path string, r io.Reader) (err error) { ospath := filepath.FromSlash(dir) if ospath != "" { - err = fs.MkdirAll(ospath, 0777) // rwx, rw, r + err = fs.MkdirAll(ospath, 0o777) // rwx, rw, r if err != nil { return } @@ -124,7 +124,7 @@ func GetTempDir(fs Fs, subPath string) string { return addSlash(dir) } - err := fs.MkdirAll(dir, 0777) + err := fs.MkdirAll(dir, 0o777) if err != nil { panic(err) } @@ -197,7 +197,6 @@ func FileContainsAnyBytes(fs Fs, filename string, subslices [][]byte) (bool, err // readerContains reports whether any of the subslices is within r. func readerContainsAny(r io.Reader, subslices ...[]byte) bool { - if r == nil || len(subslices) == 0 { return false } diff --git a/vendor/modules.txt b/vendor/modules.txt index 47e092460..3be47da12 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -279,6 +279,11 @@ github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8 github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/types github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/utils github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/utils/mocks +# github.com/k8snetworkplumbingwg/sriovnet v1.2.0 +## explicit; go 1.18 +github.com/k8snetworkplumbingwg/sriovnet +github.com/k8snetworkplumbingwg/sriovnet/pkg/utils/filesystem +github.com/k8snetworkplumbingwg/sriovnet/pkg/utils/netlinkops # github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de ## explicit github.com/liggitt/tabwriter @@ -445,7 +450,7 @@ github.com/safchain/ethtool # github.com/shopspring/decimal v1.2.0 ## explicit; go 1.13 github.com/shopspring/decimal -# github.com/spf13/afero v1.9.3 +# github.com/spf13/afero v1.9.4 ## explicit; go 1.16 github.com/spf13/afero github.com/spf13/afero/internal/common From 808907f61f59dc4cf835c6beda9cde06ef19e787 Mon Sep 17 00:00:00 2001 From: Yury Kulazhenkov Date: Fri, 3 May 2024 14:19:05 +0300 Subject: [PATCH 2/2] Add test for sriov.DiscoverSriovDevices function Signed-off-by: Yury Kulazhenkov --- go.mod | 2 +- pkg/host/internal/lib/ghw/ghw.go | 29 ++++ pkg/host/internal/lib/ghw/mock/mock_ghw.go | 88 ++++++++++ pkg/host/internal/sriov/sriov.go | 8 +- pkg/host/internal/sriov/sriov_test.go | 188 ++++++++++++++++++++- pkg/host/manager.go | 4 +- 6 files changed, 314 insertions(+), 5 deletions(-) create mode 100644 pkg/host/internal/lib/ghw/ghw.go create mode 100644 pkg/host/internal/lib/ghw/mock/mock_ghw.go diff --git a/go.mod b/go.mod index c875f75cb..3669c8565 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/google/go-cmp v0.6.0 github.com/hashicorp/go-retryablehttp v0.7.0 github.com/jaypipes/ghw v0.9.0 + github.com/jaypipes/pcidb v1.0.0 github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0 github.com/k8snetworkplumbingwg/sriov-network-device-plugin v0.0.0-20221127172732-a5a7395122e3 github.com/k8snetworkplumbingwg/sriovnet v1.2.0 @@ -90,7 +91,6 @@ require ( github.com/huandu/xstrings v1.3.2 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/jaypipes/pcidb v1.0.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/k8snetworkplumbingwg/govdpa v0.1.4 // indirect diff --git a/pkg/host/internal/lib/ghw/ghw.go b/pkg/host/internal/lib/ghw/ghw.go new file mode 100644 index 000000000..6a6829604 --- /dev/null +++ b/pkg/host/internal/lib/ghw/ghw.go @@ -0,0 +1,29 @@ +package ghw + +import ( + "github.com/jaypipes/ghw" +) + +func New() GHWLib { + return &libWrapper{} +} + +//go:generate ../../../../../bin/mockgen -destination mock/mock_ghw.go -source ghw.go +type GHWLib interface { + // PCI returns a pointer to an Info that provide methods to access info about devices + PCI() (Info, error) +} + +// Info interface provide methods to access info about devices +type Info interface { + // ListDevices returns a list of pointers to Device structs present on the + // host system + ListDevices() []*ghw.PCIDevice +} + +type libWrapper struct{} + +// PCI returns a pointer to an Info that provide methods to access info about devices +func (w *libWrapper) PCI() (Info, error) { + return ghw.PCI() +} diff --git a/pkg/host/internal/lib/ghw/mock/mock_ghw.go b/pkg/host/internal/lib/ghw/mock/mock_ghw.go new file mode 100644 index 000000000..2e2b4b5c5 --- /dev/null +++ b/pkg/host/internal/lib/ghw/mock/mock_ghw.go @@ -0,0 +1,88 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: ghw.go + +// Package mock_ghw is a generated GoMock package. +package mock_ghw + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + ghw "github.com/jaypipes/ghw" + ghw0 "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/ghw" +) + +// MockGHWLib is a mock of GHWLib interface. +type MockGHWLib struct { + ctrl *gomock.Controller + recorder *MockGHWLibMockRecorder +} + +// MockGHWLibMockRecorder is the mock recorder for MockGHWLib. +type MockGHWLibMockRecorder struct { + mock *MockGHWLib +} + +// NewMockGHWLib creates a new mock instance. +func NewMockGHWLib(ctrl *gomock.Controller) *MockGHWLib { + mock := &MockGHWLib{ctrl: ctrl} + mock.recorder = &MockGHWLibMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockGHWLib) EXPECT() *MockGHWLibMockRecorder { + return m.recorder +} + +// PCI mocks base method. +func (m *MockGHWLib) PCI() (ghw0.Info, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PCI") + ret0, _ := ret[0].(ghw0.Info) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PCI indicates an expected call of PCI. +func (mr *MockGHWLibMockRecorder) PCI() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PCI", reflect.TypeOf((*MockGHWLib)(nil).PCI)) +} + +// MockInfo is a mock of Info interface. +type MockInfo struct { + ctrl *gomock.Controller + recorder *MockInfoMockRecorder +} + +// MockInfoMockRecorder is the mock recorder for MockInfo. +type MockInfoMockRecorder struct { + mock *MockInfo +} + +// NewMockInfo creates a new mock instance. +func NewMockInfo(ctrl *gomock.Controller) *MockInfo { + mock := &MockInfo{ctrl: ctrl} + mock.recorder = &MockInfoMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockInfo) EXPECT() *MockInfoMockRecorder { + return m.recorder +} + +// ListDevices mocks base method. +func (m *MockInfo) ListDevices() []*ghw.PCIDevice { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListDevices") + ret0, _ := ret[0].([]*ghw.PCIDevice) + return ret0 +} + +// ListDevices indicates an expected call of ListDevices. +func (mr *MockInfoMockRecorder) ListDevices() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListDevices", reflect.TypeOf((*MockInfo)(nil).ListDevices)) +} diff --git a/pkg/host/internal/sriov/sriov.go b/pkg/host/internal/sriov/sriov.go index fef9f3a69..eba778358 100644 --- a/pkg/host/internal/sriov/sriov.go +++ b/pkg/host/internal/sriov/sriov.go @@ -18,6 +18,7 @@ import ( sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" dputilsPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/dputils" + ghwPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/ghw" netlinkPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink" sriovnetPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/sriovnet" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/store" @@ -40,6 +41,7 @@ type sriov struct { netlinkLib netlinkPkg.NetlinkLib dputilsLib dputilsPkg.DPUtilsLib sriovnetLib sriovnetPkg.SriovnetLib + ghwLib ghwPkg.GHWLib } func New(utilsHelper utils.CmdInterface, @@ -49,7 +51,8 @@ func New(utilsHelper utils.CmdInterface, vdpaHelper types.VdpaInterface, netlinkLib netlinkPkg.NetlinkLib, dputilsLib dputilsPkg.DPUtilsLib, - sriovnetLib sriovnetPkg.SriovnetLib) types.SriovInterface { + sriovnetLib sriovnetPkg.SriovnetLib, + ghwLib ghwPkg.GHWLib) types.SriovInterface { return &sriov{utilsHelper: utilsHelper, kernelHelper: kernelHelper, networkHelper: networkHelper, @@ -58,6 +61,7 @@ func New(utilsHelper utils.CmdInterface, netlinkLib: netlinkLib, dputilsLib: dputilsLib, sriovnetLib: sriovnetLib, + ghwLib: ghwLib, } } @@ -217,7 +221,7 @@ func (s *sriov) DiscoverSriovDevices(storeManager store.ManagerInterface) ([]sri log.Log.V(2).Info("DiscoverSriovDevices") pfList := []sriovnetworkv1.InterfaceExt{} - pci, err := ghw.PCI() + pci, err := s.ghwLib.PCI() if err != nil { return nil, fmt.Errorf("DiscoverSriovDevices(): error getting PCI info: %v", err) } diff --git a/pkg/host/internal/sriov/sriov_test.go b/pkg/host/internal/sriov/sriov_test.go index 563acb1fa..cf51091bd 100644 --- a/pkg/host/internal/sriov/sriov_test.go +++ b/pkg/host/internal/sriov/sriov_test.go @@ -7,6 +7,8 @@ import ( "syscall" "github.com/golang/mock/gomock" + "github.com/jaypipes/ghw" + "github.com/jaypipes/pcidb" "github.com/vishvananda/netlink" . "github.com/onsi/ginkgo/v2" @@ -14,6 +16,7 @@ import ( sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" dputilsMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/dputils/mock" + ghwMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/ghw/mock" netlinkMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink/mock" sriovnetMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/sriovnet/mock" hostMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/mock" @@ -29,6 +32,7 @@ var _ = Describe("SRIOV", func() { netlinkLibMock *netlinkMockPkg.MockNetlinkLib dputilsLibMock *dputilsMockPkg.MockDPUtilsLib sriovnetLibMock *sriovnetMockPkg.MockSriovnetLib + ghwLibMock *ghwMockPkg.MockGHWLib hostMock *hostMockPkg.MockHostManagerInterface storeManagerMode *hostStoreMockPkg.MockManagerInterface @@ -41,17 +45,109 @@ var _ = Describe("SRIOV", func() { netlinkLibMock = netlinkMockPkg.NewMockNetlinkLib(testCtrl) dputilsLibMock = dputilsMockPkg.NewMockDPUtilsLib(testCtrl) sriovnetLibMock = sriovnetMockPkg.NewMockSriovnetLib(testCtrl) + ghwLibMock = ghwMockPkg.NewMockGHWLib(testCtrl) hostMock = hostMockPkg.NewMockHostManagerInterface(testCtrl) storeManagerMode = hostStoreMockPkg.NewMockManagerInterface(testCtrl) - s = New(nil, hostMock, hostMock, hostMock, hostMock, netlinkLibMock, dputilsLibMock, sriovnetLibMock) + s = New(nil, hostMock, hostMock, hostMock, hostMock, netlinkLibMock, dputilsLibMock, sriovnetLibMock, ghwLibMock) }) AfterEach(func() { testCtrl.Finish() }) + Context("DiscoverSriovDevices", func() { + var ( + ghwInfoMock *ghwMockPkg.MockInfo + ) + BeforeEach(func() { + ghwInfoMock = ghwMockPkg.NewMockInfo(testCtrl) + ghwLibMock.EXPECT().PCI().Return(ghwInfoMock, nil) + origNicMap := sriovnetworkv1.NicIDMap + sriovnetworkv1.InitNicIDMapFromList([]string{ + "15b3 101d 101e", + }) + DeferCleanup(func() { + sriovnetworkv1.NicIDMap = origNicMap + }) + }) + + It("discovered", func() { + ghwInfoMock.EXPECT().ListDevices().Return(getTestPCIDevices()) + dputilsLibMock.EXPECT().IsSriovVF("0000:d8:00.0").Return(false) + dputilsLibMock.EXPECT().IsSriovVF("0000:d8:00.2").Return(true) + dputilsLibMock.EXPECT().IsSriovVF("0000:3b:00.0").Return(false) + dputilsLibMock.EXPECT().GetDriverName("0000:d8:00.0").Return("mlx5_core", nil) + hostMock.EXPECT().TryGetInterfaceName("0000:d8:00.0").Return("enp216s0f0np0") + + pfLinkMock := netlinkMockPkg.NewMockLink(testCtrl) + netlinkLibMock.EXPECT().LinkByName("enp216s0f0np0").Return(pfLinkMock, nil) + + mac, _ := net.ParseMAC("08:c0:eb:70:74:4e") + pfLinkMock.EXPECT().Attrs().Return(&netlink.LinkAttrs{ + MTU: 1500, + HardwareAddr: mac, + EncapType: "ether", + }).MinTimes(1) + hostMock.EXPECT().GetNetDevLinkSpeed("enp216s0f0np0").Return("100000 Mb/s") + storeManagerMode.EXPECT().LoadPfsStatus("0000:d8:00.0").Return(nil, false, nil) + + dputilsLibMock.EXPECT().IsSriovPF("0000:d8:00.0").Return(true) + dputilsLibMock.EXPECT().GetSriovVFcapacity("0000:d8:00.0").Return(1) + dputilsLibMock.EXPECT().GetVFconfigured("0000:d8:00.0").Return(1) + netlinkLibMock.EXPECT().DevLinkGetDeviceByName("pci", "0000:d8:00.0").Return( + &netlink.DevlinkDevice{Attrs: netlink.DevlinkDevAttrs{Eswitch: netlink.DevlinkDevEswitchAttr{Mode: "switchdev"}}}, nil) + dputilsLibMock.EXPECT().SriovConfigured("0000:d8:00.0").Return(true) + dputilsLibMock.EXPECT().GetVFList("0000:d8:00.0").Return([]string{"0000:d8:00.2"}, nil) + dputilsLibMock.EXPECT().GetDriverName("0000:d8:00.2").Return("mlx5_core", nil) + dputilsLibMock.EXPECT().GetVFID("0000:d8:00.2").Return(0, nil) + hostMock.EXPECT().DiscoverVDPAType("0000:d8:00.2").Return("") + + hostMock.EXPECT().TryGetInterfaceName("0000:d8:00.2").Return("enp216s0f0v0") + vfLinkMock := netlinkMockPkg.NewMockLink(testCtrl) + netlinkLibMock.EXPECT().LinkByName("enp216s0f0v0").Return(vfLinkMock, nil) + + mac, _ = net.ParseMAC("4e:fd:3d:08:59:b1") + vfLinkMock.EXPECT().Attrs().Return(&netlink.LinkAttrs{ + MTU: 1500, + HardwareAddr: mac, + }).MinTimes(1) + + sriovnetLibMock.EXPECT().GetVfRepresentor("enp216s0f0np0", 0).Return("enp216s0f0np0_0", nil) + + ret, err := s.DiscoverSriovDevices(storeManagerMode) + Expect(err).NotTo(HaveOccurred()) + Expect(ret).To(HaveLen(1)) + Expect(ret[0]).To(Equal(sriovnetworkv1.InterfaceExt{ + Name: "enp216s0f0np0", + Mac: "08:c0:eb:70:74:4e", + Driver: "mlx5_core", + PciAddress: "0000:d8:00.0", + Vendor: "15b3", + DeviceID: "101d", + Mtu: 1500, + NumVfs: 1, + LinkSpeed: "100000 Mb/s", + LinkType: "ETH", + EswitchMode: "switchdev", + ExternallyManaged: false, + TotalVfs: 1, + VFs: []sriovnetworkv1.VirtualFunction{{ + Name: "enp216s0f0v0", + Mac: "4e:fd:3d:08:59:b1", + Driver: "mlx5_core", + PciAddress: "0000:d8:00.2", + Vendor: "15b3", + DeviceID: "101e", + Mtu: 1500, + VfID: 0, + RepresentorName: "enp216s0f0np0_0", + }}, + })) + }) + }) + Context("SetSriovNumVfs", func() { It("set", func() { helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ @@ -437,3 +533,93 @@ var _ = Describe("SRIOV", func() { }) }) }) + +func getTestPCIDevices() []*ghw.PCIDevice { + return []*ghw.PCIDevice{{ + Driver: "mlx5_core", + Address: "0000:d8:00.0", + Vendor: &pcidb.Vendor{ + ID: "15b3", + Name: "Mellanox Technologies", + }, + Product: &pcidb.Product{ + ID: "101d", + Name: "MT2892 Family [ConnectX-6 Dx]", + }, + Revision: "0x00", + Subsystem: &pcidb.Product{ + ID: "0083", + Name: "unknown", + }, + Class: &pcidb.Class{ + ID: "02", + Name: "Network controller", + }, + Subclass: &pcidb.Subclass{ + ID: "00", + Name: "Ethernet controller", + }, + ProgrammingInterface: &pcidb.ProgrammingInterface{ + ID: "00", + Name: "unknonw", + }, + }, + { + Driver: "mlx5_core", + Address: "0000:d8:00.2", + Vendor: &pcidb.Vendor{ + ID: "15b3", + Name: "Mellanox Technologies", + }, + Product: &pcidb.Product{ + ID: "101e", + Name: "ConnectX Family mlx5Gen Virtual Function", + }, + Revision: "0x00", + Subsystem: &pcidb.Product{ + ID: "0083", + Name: "unknown", + }, + Class: &pcidb.Class{ + ID: "02", + Name: "Network controller", + }, + Subclass: &pcidb.Subclass{ + ID: "00", + Name: "Ethernet controller", + }, + ProgrammingInterface: &pcidb.ProgrammingInterface{ + ID: "00", + Name: "unknonw", + }, + }, + { + Driver: "mlx5_core", + Address: "0000:3b:00.0", + Vendor: &pcidb.Vendor{ + ID: "15b3", + Name: "Mellanox Technologies", + }, + Product: &pcidb.Product{ + ID: "aaaa", // not supported + Name: "not supported", + }, + Class: &pcidb.Class{ + ID: "02", + Name: "Network controller", + }, + }, + { + Driver: "test", + Address: "0000:d7:16.5", + Vendor: &pcidb.Vendor{ + ID: "8086", + Name: "Intel Corporation", + }, + Class: &pcidb.Class{ + ID: "11", // not network device + Name: "Signal processing controller", + }, + }, + } +} diff --git a/pkg/host/manager.go b/pkg/host/manager.go index e640f367e..de7f6e3b6 100644 --- a/pkg/host/manager.go +++ b/pkg/host/manager.go @@ -4,6 +4,7 @@ import ( "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/kernel" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/dputils" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/ethtool" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/ghw" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/sriovnet" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/network" @@ -42,12 +43,13 @@ func NewHostManager(utilsInterface utils.CmdInterface) HostManagerInterface { netlinkLib := netlink.New() ethtoolLib := ethtool.New() sriovnetLib := sriovnet.New() + ghwLib := ghw.New() k := kernel.New(utilsInterface) n := network.New(utilsInterface, dpUtils, netlinkLib, ethtoolLib) sv := service.New(utilsInterface) u := udev.New(utilsInterface) v := vdpa.New(k, netlinkLib) - sr := sriov.New(utilsInterface, k, n, u, v, netlinkLib, dpUtils, sriovnetLib) + sr := sriov.New(utilsInterface, k, n, u, v, netlinkLib, dpUtils, sriovnetLib, ghwLib) return &hostManager{ utilsInterface,