diff --git a/pkg/host/internal/lib/ghw/ghw.go b/pkg/host/internal/lib/ghw/ghw.go new file mode 100644 index 000000000..6a6829604 --- /dev/null +++ b/pkg/host/internal/lib/ghw/ghw.go @@ -0,0 +1,29 @@ +package ghw + +import ( + "github.com/jaypipes/ghw" +) + +func New() GHWLib { + return &libWrapper{} +} + +//go:generate ../../../../../bin/mockgen -destination mock/mock_ghw.go -source ghw.go +type GHWLib interface { + // PCI returns a pointer to an Info that provide methods to access info about devices + PCI() (Info, error) +} + +// Info interface provide methods to access info about devices +type Info interface { + // ListDevices returns a list of pointers to Device structs present on the + // host system + ListDevices() []*ghw.PCIDevice +} + +type libWrapper struct{} + +// PCI returns a pointer to an Info that provide methods to access info about devices +func (w *libWrapper) PCI() (Info, error) { + return ghw.PCI() +} diff --git a/pkg/host/internal/lib/ghw/mock/mock_ghw.go b/pkg/host/internal/lib/ghw/mock/mock_ghw.go new file mode 100644 index 000000000..2e2b4b5c5 --- /dev/null +++ b/pkg/host/internal/lib/ghw/mock/mock_ghw.go @@ -0,0 +1,88 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: ghw.go + +// Package mock_ghw is a generated GoMock package. +package mock_ghw + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + ghw "github.com/jaypipes/ghw" + ghw0 "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/ghw" +) + +// MockGHWLib is a mock of GHWLib interface. +type MockGHWLib struct { + ctrl *gomock.Controller + recorder *MockGHWLibMockRecorder +} + +// MockGHWLibMockRecorder is the mock recorder for MockGHWLib. +type MockGHWLibMockRecorder struct { + mock *MockGHWLib +} + +// NewMockGHWLib creates a new mock instance. +func NewMockGHWLib(ctrl *gomock.Controller) *MockGHWLib { + mock := &MockGHWLib{ctrl: ctrl} + mock.recorder = &MockGHWLibMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockGHWLib) EXPECT() *MockGHWLibMockRecorder { + return m.recorder +} + +// PCI mocks base method. +func (m *MockGHWLib) PCI() (ghw0.Info, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PCI") + ret0, _ := ret[0].(ghw0.Info) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PCI indicates an expected call of PCI. +func (mr *MockGHWLibMockRecorder) PCI() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PCI", reflect.TypeOf((*MockGHWLib)(nil).PCI)) +} + +// MockInfo is a mock of Info interface. +type MockInfo struct { + ctrl *gomock.Controller + recorder *MockInfoMockRecorder +} + +// MockInfoMockRecorder is the mock recorder for MockInfo. +type MockInfoMockRecorder struct { + mock *MockInfo +} + +// NewMockInfo creates a new mock instance. +func NewMockInfo(ctrl *gomock.Controller) *MockInfo { + mock := &MockInfo{ctrl: ctrl} + mock.recorder = &MockInfoMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockInfo) EXPECT() *MockInfoMockRecorder { + return m.recorder +} + +// ListDevices mocks base method. +func (m *MockInfo) ListDevices() []*ghw.PCIDevice { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListDevices") + ret0, _ := ret[0].([]*ghw.PCIDevice) + return ret0 +} + +// ListDevices indicates an expected call of ListDevices. +func (mr *MockInfoMockRecorder) ListDevices() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListDevices", reflect.TypeOf((*MockInfo)(nil).ListDevices)) +} diff --git a/pkg/host/internal/sriov/sriov.go b/pkg/host/internal/sriov/sriov.go index fef9f3a69..eba778358 100644 --- a/pkg/host/internal/sriov/sriov.go +++ b/pkg/host/internal/sriov/sriov.go @@ -18,6 +18,7 @@ import ( sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" dputilsPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/dputils" + ghwPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/ghw" netlinkPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink" sriovnetPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/sriovnet" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/store" @@ -40,6 +41,7 @@ type sriov struct { netlinkLib netlinkPkg.NetlinkLib dputilsLib dputilsPkg.DPUtilsLib sriovnetLib sriovnetPkg.SriovnetLib + ghwLib ghwPkg.GHWLib } func New(utilsHelper utils.CmdInterface, @@ -49,7 +51,8 @@ func New(utilsHelper utils.CmdInterface, vdpaHelper types.VdpaInterface, netlinkLib netlinkPkg.NetlinkLib, dputilsLib dputilsPkg.DPUtilsLib, - sriovnetLib sriovnetPkg.SriovnetLib) types.SriovInterface { + sriovnetLib sriovnetPkg.SriovnetLib, + ghwLib ghwPkg.GHWLib) types.SriovInterface { return &sriov{utilsHelper: utilsHelper, kernelHelper: kernelHelper, networkHelper: networkHelper, @@ -58,6 +61,7 @@ func New(utilsHelper utils.CmdInterface, netlinkLib: netlinkLib, dputilsLib: dputilsLib, sriovnetLib: sriovnetLib, + ghwLib: ghwLib, } } @@ -217,7 +221,7 @@ func (s *sriov) DiscoverSriovDevices(storeManager store.ManagerInterface) ([]sri log.Log.V(2).Info("DiscoverSriovDevices") pfList := []sriovnetworkv1.InterfaceExt{} - pci, err := ghw.PCI() + pci, err := s.ghwLib.PCI() if err != nil { return nil, fmt.Errorf("DiscoverSriovDevices(): error getting PCI info: %v", err) } diff --git a/pkg/host/internal/sriov/sriov_test.go b/pkg/host/internal/sriov/sriov_test.go index 563acb1fa..cf51091bd 100644 --- a/pkg/host/internal/sriov/sriov_test.go +++ b/pkg/host/internal/sriov/sriov_test.go @@ -7,6 +7,8 @@ import ( "syscall" "github.com/golang/mock/gomock" + "github.com/jaypipes/ghw" + "github.com/jaypipes/pcidb" "github.com/vishvananda/netlink" . "github.com/onsi/ginkgo/v2" @@ -14,6 +16,7 @@ import ( sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" dputilsMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/dputils/mock" + ghwMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/ghw/mock" netlinkMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink/mock" sriovnetMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/sriovnet/mock" hostMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/mock" @@ -29,6 +32,7 @@ var _ = Describe("SRIOV", func() { netlinkLibMock *netlinkMockPkg.MockNetlinkLib dputilsLibMock *dputilsMockPkg.MockDPUtilsLib sriovnetLibMock *sriovnetMockPkg.MockSriovnetLib + ghwLibMock *ghwMockPkg.MockGHWLib hostMock *hostMockPkg.MockHostManagerInterface storeManagerMode *hostStoreMockPkg.MockManagerInterface @@ -41,17 +45,109 @@ var _ = Describe("SRIOV", func() { netlinkLibMock = netlinkMockPkg.NewMockNetlinkLib(testCtrl) dputilsLibMock = dputilsMockPkg.NewMockDPUtilsLib(testCtrl) sriovnetLibMock = sriovnetMockPkg.NewMockSriovnetLib(testCtrl) + ghwLibMock = ghwMockPkg.NewMockGHWLib(testCtrl) hostMock = hostMockPkg.NewMockHostManagerInterface(testCtrl) storeManagerMode = hostStoreMockPkg.NewMockManagerInterface(testCtrl) - s = New(nil, hostMock, hostMock, hostMock, hostMock, netlinkLibMock, dputilsLibMock, sriovnetLibMock) + s = New(nil, hostMock, hostMock, hostMock, hostMock, netlinkLibMock, dputilsLibMock, sriovnetLibMock, ghwLibMock) }) AfterEach(func() { testCtrl.Finish() }) + Context("DiscoverSriovDevices", func() { + var ( + ghwInfoMock *ghwMockPkg.MockInfo + ) + BeforeEach(func() { + ghwInfoMock = ghwMockPkg.NewMockInfo(testCtrl) + ghwLibMock.EXPECT().PCI().Return(ghwInfoMock, nil) + origNicMap := sriovnetworkv1.NicIDMap + sriovnetworkv1.InitNicIDMapFromList([]string{ + "15b3 101d 101e", + }) + DeferCleanup(func() { + sriovnetworkv1.NicIDMap = origNicMap + }) + }) + + It("discovered", func() { + ghwInfoMock.EXPECT().ListDevices().Return(getTestPCIDevices()) + dputilsLibMock.EXPECT().IsSriovVF("0000:d8:00.0").Return(false) + dputilsLibMock.EXPECT().IsSriovVF("0000:d8:00.2").Return(true) + dputilsLibMock.EXPECT().IsSriovVF("0000:3b:00.0").Return(false) + dputilsLibMock.EXPECT().GetDriverName("0000:d8:00.0").Return("mlx5_core", nil) + hostMock.EXPECT().TryGetInterfaceName("0000:d8:00.0").Return("enp216s0f0np0") + + pfLinkMock := netlinkMockPkg.NewMockLink(testCtrl) + netlinkLibMock.EXPECT().LinkByName("enp216s0f0np0").Return(pfLinkMock, nil) + + mac, _ := net.ParseMAC("08:c0:eb:70:74:4e") + pfLinkMock.EXPECT().Attrs().Return(&netlink.LinkAttrs{ + MTU: 1500, + HardwareAddr: mac, + EncapType: "ether", + }).MinTimes(1) + hostMock.EXPECT().GetNetDevLinkSpeed("enp216s0f0np0").Return("100000 Mb/s") + storeManagerMode.EXPECT().LoadPfsStatus("0000:d8:00.0").Return(nil, false, nil) + + dputilsLibMock.EXPECT().IsSriovPF("0000:d8:00.0").Return(true) + dputilsLibMock.EXPECT().GetSriovVFcapacity("0000:d8:00.0").Return(1) + dputilsLibMock.EXPECT().GetVFconfigured("0000:d8:00.0").Return(1) + netlinkLibMock.EXPECT().DevLinkGetDeviceByName("pci", "0000:d8:00.0").Return( + &netlink.DevlinkDevice{Attrs: netlink.DevlinkDevAttrs{Eswitch: netlink.DevlinkDevEswitchAttr{Mode: "switchdev"}}}, nil) + dputilsLibMock.EXPECT().SriovConfigured("0000:d8:00.0").Return(true) + dputilsLibMock.EXPECT().GetVFList("0000:d8:00.0").Return([]string{"0000:d8:00.2"}, nil) + dputilsLibMock.EXPECT().GetDriverName("0000:d8:00.2").Return("mlx5_core", nil) + dputilsLibMock.EXPECT().GetVFID("0000:d8:00.2").Return(0, nil) + hostMock.EXPECT().DiscoverVDPAType("0000:d8:00.2").Return("") + + hostMock.EXPECT().TryGetInterfaceName("0000:d8:00.2").Return("enp216s0f0v0") + vfLinkMock := netlinkMockPkg.NewMockLink(testCtrl) + netlinkLibMock.EXPECT().LinkByName("enp216s0f0v0").Return(vfLinkMock, nil) + + mac, _ = net.ParseMAC("4e:fd:3d:08:59:b1") + vfLinkMock.EXPECT().Attrs().Return(&netlink.LinkAttrs{ + MTU: 1500, + HardwareAddr: mac, + }).MinTimes(1) + + sriovnetLibMock.EXPECT().GetVfRepresentor("enp216s0f0np0", 0).Return("enp216s0f0np0_0", nil) + + ret, err := s.DiscoverSriovDevices(storeManagerMode) + Expect(err).NotTo(HaveOccurred()) + Expect(ret).To(HaveLen(1)) + Expect(ret[0]).To(Equal(sriovnetworkv1.InterfaceExt{ + Name: "enp216s0f0np0", + Mac: "08:c0:eb:70:74:4e", + Driver: "mlx5_core", + PciAddress: "0000:d8:00.0", + Vendor: "15b3", + DeviceID: "101d", + Mtu: 1500, + NumVfs: 1, + LinkSpeed: "100000 Mb/s", + LinkType: "ETH", + EswitchMode: "switchdev", + ExternallyManaged: false, + TotalVfs: 1, + VFs: []sriovnetworkv1.VirtualFunction{{ + Name: "enp216s0f0v0", + Mac: "4e:fd:3d:08:59:b1", + Driver: "mlx5_core", + PciAddress: "0000:d8:00.2", + Vendor: "15b3", + DeviceID: "101e", + Mtu: 1500, + VfID: 0, + RepresentorName: "enp216s0f0np0_0", + }}, + })) + }) + }) + Context("SetSriovNumVfs", func() { It("set", func() { helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ @@ -437,3 +533,93 @@ var _ = Describe("SRIOV", func() { }) }) }) + +func getTestPCIDevices() []*ghw.PCIDevice { + return []*ghw.PCIDevice{{ + Driver: "mlx5_core", + Address: "0000:d8:00.0", + Vendor: &pcidb.Vendor{ + ID: "15b3", + Name: "Mellanox Technologies", + }, + Product: &pcidb.Product{ + ID: "101d", + Name: "MT2892 Family [ConnectX-6 Dx]", + }, + Revision: "0x00", + Subsystem: &pcidb.Product{ + ID: "0083", + Name: "unknown", + }, + Class: &pcidb.Class{ + ID: "02", + Name: "Network controller", + }, + Subclass: &pcidb.Subclass{ + ID: "00", + Name: "Ethernet controller", + }, + ProgrammingInterface: &pcidb.ProgrammingInterface{ + ID: "00", + Name: "unknonw", + }, + }, + { + Driver: "mlx5_core", + Address: "0000:d8:00.2", + Vendor: &pcidb.Vendor{ + ID: "15b3", + Name: "Mellanox Technologies", + }, + Product: &pcidb.Product{ + ID: "101e", + Name: "ConnectX Family mlx5Gen Virtual Function", + }, + Revision: "0x00", + Subsystem: &pcidb.Product{ + ID: "0083", + Name: "unknown", + }, + Class: &pcidb.Class{ + ID: "02", + Name: "Network controller", + }, + Subclass: &pcidb.Subclass{ + ID: "00", + Name: "Ethernet controller", + }, + ProgrammingInterface: &pcidb.ProgrammingInterface{ + ID: "00", + Name: "unknonw", + }, + }, + { + Driver: "mlx5_core", + Address: "0000:3b:00.0", + Vendor: &pcidb.Vendor{ + ID: "15b3", + Name: "Mellanox Technologies", + }, + Product: &pcidb.Product{ + ID: "aaaa", // not supported + Name: "not supported", + }, + Class: &pcidb.Class{ + ID: "02", + Name: "Network controller", + }, + }, + { + Driver: "test", + Address: "0000:d7:16.5", + Vendor: &pcidb.Vendor{ + ID: "8086", + Name: "Intel Corporation", + }, + Class: &pcidb.Class{ + ID: "11", // not network device + Name: "Signal processing controller", + }, + }, + } +} diff --git a/pkg/host/manager.go b/pkg/host/manager.go index e640f367e..de7f6e3b6 100644 --- a/pkg/host/manager.go +++ b/pkg/host/manager.go @@ -4,6 +4,7 @@ import ( "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/kernel" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/dputils" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/ethtool" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/ghw" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/sriovnet" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/network" @@ -42,12 +43,13 @@ func NewHostManager(utilsInterface utils.CmdInterface) HostManagerInterface { netlinkLib := netlink.New() ethtoolLib := ethtool.New() sriovnetLib := sriovnet.New() + ghwLib := ghw.New() k := kernel.New(utilsInterface) n := network.New(utilsInterface, dpUtils, netlinkLib, ethtoolLib) sv := service.New(utilsInterface) u := udev.New(utilsInterface) v := vdpa.New(k, netlinkLib) - sr := sriov.New(utilsInterface, k, n, u, v, netlinkLib, dpUtils, sriovnetLib) + sr := sriov.New(utilsInterface, k, n, u, v, netlinkLib, dpUtils, sriovnetLib, ghwLib) return &hostManager{ utilsInterface,